<commit_before>import time
import os
from datetime import datetime

from biothings.utils.mongo import get_src_dump
from biothings.utils.common import timesofar


class BaseDumper(object):
    # override in subclass accordingly
    SRC_NAME = None
    SRC_ROOT_FOLDER = None #se  source folder (without version/dates)

    def __init__(self, no_confirm=True, archive=True):
        self.client = None
        self.logger = None
        self.src_dump = None
        self.src_doc = None
        self.no_confirm = no_confirm
        self.archive = archive
        self.to_dump = []
        self.release = None
        self.t0 = time.time()
        self.logfile = None
        self.prev_data_folder = None
        self.timestamp = time.strftime('%Y%m%d')
        # init
        self.setup_log()
        self.prepare()

    def create_todump_list(self,force=False):
        """Fill self.to_dump list with dict("remote":remote_path,"local":local_path)
        elements. This is the todo list for the dumper. It's a good place to
        check whether needs to be downloaded. If 'force' is True though, all files
        will be considered for download"""
        raise NotImplementedError("Define in subclass")

    def prepare_client(self):
        """do initialization to make the client ready to dump files"""
        raise NotImplementedError("Define in subclass")

    def need_prepare(self):
        """check whether some prepare step should executed
        before running dump"""

    def release_client(self):
        """Do whatever necessary (like closing network connection)
        to "release" the client"""
        raise NotImplementedError("Define in subclass")

    def remote_is_better(self,remotefile,localfile):
        '''Compared to local file, check if remote file is worth downloading.
        (like either bigger or newer for instance)'''
        raise NotImplementedError("Define in subclass")

    def download(self,remotefile,localfile):
        """Download "remotefile' to local location defined by 'localfile'"""
        raise NotImplementedError("Define in subclass")

    def setup_log(self):
        import logging as logging_mod
        if not os.path.exists(self.SRC_ROOT_FOLDER):
            os.makedirs(self.SRC_ROOT_FOLDER)
        self.logfile = os.path.join(self.SRC_ROOT_FOLDER, '%s_%s_dump.log' % (self.SRC_NAME,self.timestamp))
        fh = logging_mod.FileHandler(self.logfile)
        fh.setFormatter(logging_mod.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
        fh.name = "logfile"
        sh = logging_mod.StreamHandler()
        sh.name = "logstream"
        self.logger = logging_mod.getLogger("%s_dump" % self.SRC_NAME)
        self.logger.setLevel(logging_mod.DEBUG)
        if not fh.name in [h.name for h in self.logger.handlers]:
            self.logger.addHandler(fh)
        if not sh.name in [h.name for h in self.logger.handlers]:
            self.logger.addHandler(sh)

    def prepare(self):
        self.prepare_client()
        self.prepare_src_dump()
        self.new_data_folder = self.get_new_data_folder()
        self.current_data_folder = self.src_doc.get("data_folder")
        self.to_dump = []

    def prepare_src_dump(self):
        # Mongo side
        self.src_dump = get_src_dump()
        self.src_doc = self.src_dump.find_one({'_id': self.SRC_NAME}) or {}

    def register_status(self,status,transient=False,**extra):
        self.src_doc = {
                '_id': self.SRC_NAME,
               'data_folder': self.new_data_folder,
               'release': self.release,
               'download' : {
                   'logfile': self.logfile,
                   'started_at': datetime.now(),
                   'status': status}
               }
        # only register time when it's a final state
        if transient:
            self.src_doc["download"]["pid"] = os.getpid()
        else:
            self.src_doc["download"]["time"] = timesofar(self.t0)
        if "download" in extra:
            self.src_doc["download"].update(extra["download"])
        else:
            self.src_doc.update(extra)
        self.src_dump.save(self.src_doc)<commit_msg>fixed data_folder when run for the 1st time (no previous record in db)<commit_after>import time
import os
from datetime import datetime

from biothings.utils.mongo import get_src_dump
from biothings.utils.common import timesofar


class BaseDumper(object):
    # override in subclass accordingly
    SRC_NAME = None
    SRC_ROOT_FOLDER = None #se  source folder (without version/dates)

    def __init__(self, no_confirm=True, archive=True):
        self.client = None
        self.logger = None
        self.src_dump = None
        self.src_doc = None
        self.no_confirm = no_confirm
        self.archive = archive
        self.to_dump = []
        self.release = None
        self.t0 = time.time()
        self.logfile = None
        self.prev_data_folder = None
        self.timestamp = time.strftime('%Y%m%d')
        # init
        self.setup_log()
        self.prepare()

    def create_todump_list(self,force=False):
        """Fill self.to_dump list with dict("remote":remote_path,"local":local_path)
        elements. This is the todo list for the dumper. It's a good place to
        check whether needs to be downloaded. If 'force' is True though, all files
        will be considered for download"""
        raise NotImplementedError("Define in subclass")

    def prepare_client(self):
        """do initialization to make the client ready to dump files"""
        raise NotImplementedError("Define in subclass")

    def need_prepare(self):
        """check whether some prepare step should executed
        before running dump"""

    def release_client(self):
        """Do whatever necessary (like closing network connection)
        to "release" the client"""
        raise NotImplementedError("Define in subclass")

    def remote_is_better(self,remotefile,localfile):
        '''Compared to local file, check if remote file is worth downloading.
        (like either bigger or newer for instance)'''
        raise NotImplementedError("Define in subclass")

    def download(self,remotefile,localfile):
        """Download "remotefile' to local location defined by 'localfile'"""
        raise NotImplementedError("Define in subclass")

    def setup_log(self):
        import logging as logging_mod
        if not os.path.exists(self.SRC_ROOT_FOLDER):
            os.makedirs(self.SRC_ROOT_FOLDER)
        self.logfile = os.path.join(self.SRC_ROOT_FOLDER, '%s_%s_dump.log' % (self.SRC_NAME,self.timestamp))
        fh = logging_mod.FileHandler(self.logfile)
        fh.setFormatter(logging_mod.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
        fh.name = "logfile"
        sh = logging_mod.StreamHandler()
        sh.name = "logstream"
        self.logger = logging_mod.getLogger("%s_dump" % self.SRC_NAME)
        self.logger.setLevel(logging_mod.DEBUG)
        if not fh.name in [h.name for h in self.logger.handlers]:
            self.logger.addHandler(fh)
        if not sh.name in [h.name for h in self.logger.handlers]:
            self.logger.addHandler(sh)

    def prepare(self):
        self.prepare_client()
        self.prepare_src_dump()
        self.new_data_folder = self.get_new_data_folder()
        self.current_data_folder = self.src_doc.get("data_folder") or self.new_data_folder
        self.to_dump = []

    def prepare_src_dump(self):
        # Mongo side
        self.src_dump = get_src_dump()
        self.src_doc = self.src_dump.find_one({'_id': self.SRC_NAME}) or {}

    def register_status(self,status,transient=False,**extra):
        self.src_doc = {
                '_id': self.SRC_NAME,
               'data_folder': self.new_data_folder,
               'release': self.release,
               'download' : {
                   'logfile': self.logfile,
                   'started_at': datetime.now(),
                   'status': status}
               }
        # only register time when it's a final state
        if transient:
            self.src_doc["download"]["pid"] = os.getpid()
        else:
            self.src_doc["download"]["time"] = timesofar(self.t0)
        if "download" in extra:
            self.src_doc["download"].update(extra["download"])
        else:
            self.src_doc.update(extra)
        self.src_dump.save(self.src_doc)
{'fraction_non_alphanumeric': 0.1044171925229194, 'fraction_numerical': 0.0005953089653530182, 'mean_word_length': 2.4941763727121464, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '214657', 'n_tokens_mistral': 2513, 'n_tokens_neox': 2443, 'n_words': 603}