<commit_before>import os, sys, types
import asyncio
import logging
from pprint import pformat
import importlib

from biothings.utils.hub_db import get_data_plugin
from biothings.utils.dataload import to_boolean
from biothings.utils.manager import BaseSourceManager
from biothings.utils.hub_db import get_src_master, get_source_fullname, \
                                   get_src_dump


class SourceManager(BaseSourceManager):
    """
    Helper class to get information about a datasource,
    whether it has a dumper and/or uploaders associated.
    """

    def __init__(self, source_list, dump_manager, upload_manager):
        self._orig_source_list = source_list
        self.source_list = None
        self.dump_manager = dump_manager
        self.upload_manager = upload_manager
        self.reload()
        self.src_master = get_src_master()
        self.src_dump = get_src_dump()
        # honoring BaseSourceManager interface (gloups...-
        self.register = {}

    def reload(self):
        # clear registers
        self.dump_manager.register.clear()
        self.upload_manager.register.clear()
        # re-eval source list (so if it's a string, it'll re-discover sources)
        self.source_list = self.find_sources(self._orig_source_list)
        self.dump_manager.register_sources(self.source_list)
        self.upload_manager.register_sources(self.source_list)

    def find_sources(self,paths):
        sources = []

        if not type(paths) == list:
            paths = [paths]

        def eval_one_src(one_path):
            if "/" in one_path:
                # it's path to directory
                # expecting 
                if not one_path in sys.path:
                    logging.info("Adding '%s' to python path" % one_path)
                    sys.path.insert(0,one_path)
                for d in os.listdir(one_path):
                    if d.endswith("__pycache__"):
                        continue
                    sources.append(d)
            else:
                # assuming it's path to a python module (oath.to.module)
                sources.append(one_path)

        def eval_one_root(root):
            logging.debug("Discovering sources in %s" % root)
            # root is a module path where sources can be found
            rootdir,__init__ = os.path.split(root.__file__)
            for srcdir in os.listdir(rootdir):
                if srcdir.endswith("__pycache__"):
                    continue
                srcpath = os.path.join(rootdir,srcdir)
                if os.path.isdir(srcpath):
                    srcmod_str = "%s.%s" % (root.__name__,srcdir)
                    sources.append(srcmod_str)<commit_msg>fix typo<commit_after>import os, sys, types
import asyncio
import logging
from pprint import pformat
import importlib

from biothings.utils.hub_db import get_data_plugin
from biothings.utils.dataload import to_boolean
from biothings.utils.manager import BaseSourceManager
from biothings.utils.hub_db import get_src_master, get_source_fullname, \
                                   get_src_dump


class SourceManager(BaseSourceManager):
    """
    Helper class to get information about a datasource,
    whether it has a dumper and/or uploaders associated.
    """

    def __init__(self, source_list, dump_manager, upload_manager):
        self._orig_source_list = source_list
        self.source_list = None
        self.dump_manager = dump_manager
        self.upload_manager = upload_manager
        self.reload()
        self.src_master = get_src_master()
        self.src_dump = get_src_dump()
        # honoring BaseSourceManager interface (gloups...-
        self.register = {}

    def reload(self):
        # clear registers
        self.dump_manager.register.clear()
        self.upload_manager.register.clear()
        # re-eval source list (so if it's a string, it'll re-discover sources)
        self.source_list = self.find_sources(self._orig_source_list)
        self.dump_manager.register_sources(self.source_list)
        self.upload_manager.register_sources(self.source_list)

    def find_sources(self,paths):
        sources = []

        if not type(paths) == list:
            paths = [paths]

        def eval_one_source(one_path):
            if "/" in one_path:
                # it's path to directory
                # expecting 
                if not one_path in sys.path:
                    logging.info("Adding '%s' to python path" % one_path)
                    sys.path.insert(0,one_path)
                for d in os.listdir(one_path):
                    if d.endswith("__pycache__"):
                        continue
                    sources.append(d)
            else:
                # assuming it's path to a python module (oath.to.module)
                sources.append(one_path)

        def eval_one_root(root):
            logging.debug("Discovering sources in %s" % root)
            # root is a module path where sources can be found
            rootdir,__init__ = os.path.split(root.__file__)
            for srcdir in os.listdir(rootdir):
                if srcdir.endswith("__pycache__"):
                    continue
                srcpath = os.path.join(rootdir,srcdir)
                if os.path.isdir(srcpath):
                    srcmod_str = "%s.%s" % (root.__name__,srcdir)
                    sources.append(srcmod_str)
{'fraction_non_alphanumeric': 0.08128262490678598, 'fraction_numerical': 0.0003728560775540641, 'mean_word_length': 1.9938616071428572, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4129474', 'n_tokens_mistral': 1539, 'n_tokens_neox': 1473, 'n_words': 354}