# Path: dlstats/constants.py
# CACHE_URL = os.environ.get('WIDUKIND_CACHE_URL', 'simple') #redis://localhost:6379/0
# SCHEMAS_VALIDATION_DISABLE = os.environ.get('WIDUKIND_SCHEMAS_VALIDATION_DISABLE', 'false')
# 
# Path: dlstats/fetchers/_commons.py
# class Categories(DlstatsCollection):
#     """Categories class
#     
#     Inherit from :class:`DlstatsCollection`
#     """
#     
#     def __init__(self,
#                  provider_name=None,
#                  category_code=None,
#                  name=None,
#                  position=0,
#                  parent=None,
#                  all_parents=None,
#                  datasets=[],
#                  doc_href=None,
#                  metadata=None,
#                  fetcher=None,
#                  **kwargs):
#         """
#         :param str provider_name: Provider name
#         :param str category_code: Unique Category Code
#         :param str name: Category name
#         :param list datasets: Array of dataset_code        
#         :param str doc_href: (Optional) Category - web link
#         :param Fetcher fetcher: Fetcher instance
#         """        
#         super().__init__(fetcher=fetcher)
#         self.provider_name = provider_name
#         if not provider_name:
#             self.provider_name = self.fetcher.provider_name
#         
#         self.category_code = category_code
#         self.name = name
#         self.position = position
#         self.parent = parent
#         self.all_parents = all_parents
#         self.datasets = datasets
#         self.doc_href = doc_href
#         self.metadata = metadata
# 
#         self.enable = True
#         self.lock = False
# 
#     def slug(self):
#         txt = "-".join([self.provider_name, self.category_code])
#         return slugify(txt, word_boundary=False, save_order=True)
# 
#     @property
#     def bson(self):
#         return {'provider_name': self.provider_name,
#                 'category_code': self.category_code,
#                 'name': self.name,
#                 'position': self.position,
#                 'parent': self.parent,
#                 'all_parents': self.all_parents,
#                 'slug': self.slug(),
#                 'datasets': self.datasets,
#                 'doc_href': self.doc_href,
#                 'metadata': self.metadata,
#                 "enable": self.enable,
#                 "lock": self.lock}
# 
#     @classmethod
#     def categories(cls, provider_name, db=None, **query):
#         db = db or get_mongo_db()
#         if not "provider_name" in query:
#             query["provider_name"] = provider_name
#         cursor = db[constants.COL_CATEGORIES].find(query)
#         return dict([(doc["category_code"], doc) for doc in cursor])
# 
#     @classmethod
#     def count(cls, provider_name, db=None):
#         db = db or get_mongo_db()
#         query = {"provider_name": provider_name}
#         return db[constants.COL_CATEGORIES].count(query)
# 
#     @classmethod
#     def remove_all(cls, provider_name, db=None):
#         db = db or get_mongo_db()
#         query = {"provider_name": provider_name}
#         logger.info("remove all categories for [%s]" % provider_name)
#         return db[constants.COL_CATEGORIES].remove(query)
#     
#     @classmethod
#     def search_category_for_dataset(cls, provider_name, dataset_code, db=None):
#         db = db or get_mongo_db()
#         query = {"provider_name": provider_name,
#                  "datasets.0": {"$exists": True},
#                  "datasets.dataset_code": dataset_code}
#         return db[constants.COL_CATEGORIES].find_one(query)
# 
#     @classmethod
#     def root_categories(cls, provider_name, db=None):
#         db = db or get_mongo_db()
#         query = {"provider_name": provider_name, "parent": None}
#         cursor = db[constants.COL_CATEGORIES].find(query)
#         return cursor.sort([("position", 1), ("category_code", 1)])
#     
#     @classmethod
#     def _iter_parent(cls, category, categories):
#         if not categories:
#             categories = cls.categories()
#         parents_keys = []
#         if category.get("parent"):
#             parent_id = category.get("parent")
#             parents_keys.append(parent_id)
#             parent_category = categories.get(parent_id)
#             if parent_category:
#                 parents_keys.extend(cls._iter_parent(parent_category, 
#                                                      categories))
#         return parents_keys
# 
#     @classmethod
#     def iter_parent(cls, category, categories):
#         """Recursive function for retrieve all parents
#         for one category
#         
#         Return array of category_code for all parents
#         """
#         parents_keys = cls._iter_parent(category, categories)
#         parents_keys.reverse()
#         return parents_keys
#         
#     def update_database(self):
#         schemas.category_schema(self.bson)
#         return self.update_mongo_collection(constants.COL_CATEGORIES, 
#                                             ['slug'],
#                                             self.bson)
# 
# Path: dlstats/tests/base.py
# RESOURCES_DIR = os.path.abspath(os.path.dirname(resources.__file__))


import logging
import io
import httpretty
from pprint import pprint
from slugify import slugify
from dlstats import constants
from dlstats.fetchers._commons import Categories
from dlstats.tests.base import BaseDBTestCase
# -*- coding: utf-8 -*-




#TODO: use tests.utils
def body_generator(filepath):
    '''body for large file'''
    if isinstance(filepath, io.BufferedIOBase):
        for line in filepath.readline():
            yield line
    elif isinstance(filepath, bytes):
        yield filepath
    else:
        with open(filepath, 'rb') as fp:
            for line in fp:
                yield line

def slugify_dict_keys(**kwargs):
    result = {}
    for k, v in kwargs.items():
        result[slugify(k, word_boundary=False, save_order=True)] = slugify(v, word_boundary=False, save_order=True)
    return result


# Path: dlstats/constants.py
# CACHE_URL = os.environ.get('WIDUKIND_CACHE_URL', 'simple') #redis://localhost:6379/0
# SCHEMAS_VALIDATION_DISABLE = os.environ.get('WIDUKIND_SCHEMAS_VALIDATION_DISABLE', 'false')
# 
# Path: dlstats/fetchers/_commons.py
# class Categories(DlstatsCollection):
#     """Categories class
#     
#     Inherit from :class:`DlstatsCollection`
#     """
#     
#     def __init__(self,
#                  provider_name=None,
#                  category_code=None,
#                  name=None,
#                  position=0,
#                  parent=None,
#                  all_parents=None,
#                  datasets=[],
#                  doc_href=None,
#                  metadata=None,
#                  fetcher=None,
#                  **kwargs):
#         """
#         :param str provider_name: Provider name
#         :param str category_code: Unique Category Code
#         :param str name: Category name
#         :param list datasets: Array of dataset_code        
#         :param str doc_href: (Optional) Category - web link
#         :param Fetcher fetcher: Fetcher instance
#         """        
#         super().__init__(fetcher=fetcher)
#         self.provider_name = provider_name
#         if not provider_name:
#             self.provider_name = self.fetcher.provider_name
#         
#         self.category_code = category_code
#         self.name = name
#         self.position = position
#         self.parent = parent
#         self.all_parents = all_parents
#         self.datasets = datasets
#         self.doc_href = doc_href
#         self.metadata = metadata
# 
#         self.enable = True
#         self.lock = False
# 
#     def slug(self):
#         txt = "-".join([self.provider_name, self.category_code])
#         return slugify(txt, word_boundary=False, save_order=True)
# 
#     @property
#     def bson(self):
#         return {'provider_name': self.provider_name,
#                 'category_code': self.category_code,
#                 'name': self.name,
#                 'position': self.position,
#                 'parent': self.parent,
#                 'all_parents': self.all_parents,
#                 'slug': self.slug(),
#                 'datasets': self.datasets,
#                 'doc_href': self.doc_href,
#                 'metadata': self.metadata,
#                 "enable": self.enable,
#                 "lock": self.lock}
# 
#     @classmethod
#     def categories(cls, provider_name, db=None, **query):
#         db = db or get_mongo_db()
#         if not "provider_name" in query:
#             query["provider_name"] = provider_name
#         cursor = db[constants.COL_CATEGORIES].find(query)
#         return dict([(doc["category_code"], doc) for doc in cursor])
# 
#     @classmethod
#     def count(cls, provider_name, db=None):
#         db = db or get_mongo_db()
#         query = {"provider_name": provider_name}
#         return db[constants.COL_CATEGORIES].count(query)
# 
#     @classmethod
#     def remove_all(cls, provider_name, db=None):
#         db = db or get_mongo_db()
#         query = {"provider_name": provider_name}
#         logger.info("remove all categories for [%s]" % provider_name)
#         return db[constants.COL_CATEGORIES].remove(query)
#     
#     @classmethod
#     def search_category_for_dataset(cls, provider_name, dataset_code, db=None):
#         db = db or get_mongo_db()
#         query = {"provider_name": provider_name,
#                  "datasets.0": {"$exists": True},
#                  "datasets.dataset_code": dataset_code}
#         return db[constants.COL_CATEGORIES].find_one(query)
# 
#     @classmethod
#     def root_categories(cls, provider_name, db=None):
#         db = db or get_mongo_db()
#         query = {"provider_name": provider_name, "parent": None}
#         cursor = db[constants.COL_CATEGORIES].find(query)
#         return cursor.sort([("position", 1), ("category_code", 1)])
#     
#     @classmethod
#     def _iter_parent(cls, category, categories):
#         if not categories:
#             categories = cls.categories()
#         parents_keys = []
#         if category.get("parent"):
#             parent_id = category.get("parent")
#             parents_keys.append(parent_id)
#             parent_category = categories.get(parent_id)
#             if parent_category:
#                 parents_keys.extend(cls._iter_parent(parent_category, 
#                                                      categories))
#         return parents_keys
# 
#     @classmethod
#     def iter_parent(cls, category, categories):
#         """Recursive function for retrieve all parents
#         for one category
#         
#         Return array of category_code for all parents
#         """
#         parents_keys = cls._iter_parent(category, categories)
#         parents_keys.reverse()
#         return parents_keys
#         
#     def update_database(self):
#         schemas.category_schema(self.bson)
#         return self.update_mongo_collection(constants.COL_CATEGORIES, 
#                                             ['slug'],
#                                             self.bson)
# 
# Path: dlstats/tests/base.py
# RESOURCES_DIR = os.path.abspath(os.path.dirname(resources.__file__))


# Path: dlstats/tests/fetchers/base.py
import logging
import io
import httpretty
from pprint import pprint
from slugify import slugify
from dlstats import constants
from dlstats.fetchers._commons import Categories
from dlstats.tests.base import BaseDBTestCase

# -*- coding: utf-8 -*-




#TODO: use tests.utils
def body_generator(filepath):
    '''body for large file'''
    if isinstance(filepath, io.BufferedIOBase):
        for line in filepath.readline():
            yield line
    elif isinstance(filepath, bytes):
        yield filepath
    else:
        with open(filepath, 'rb') as fp:
            for line in fp:
                yield line

def slugify_dict_keys(**kwargs):
    result = {}
    for k, v in kwargs.items():
        result[slugify(k, word_boundary=False, save_order=True)] = slugify(v, word_boundary=False, save_order=True)
    return result


class BaseFetcherTestCase(BaseDBTestCase):