<commit_before># -*- coding:utf-8 -*-
__author__ = 'nathan'

import threading
import logging
import time
import codecs

class BaseSync():

    def __init__(self):
        pass

    def setup(self):
        pass

    def partition(self):
        pass

    def merge(self):
        pass

    def target(self):
        pass

    def sync(self, target):
        pass

    def teardown(self):
        pass

class Dumper(threading.Thread):
    def __init__(self, dst_file, sq):
        threading.Thread.__init__(self)
        self._logger = logging.getLogger("dbsync.syncs")
        self.dst_file = dst_file
        self.sq = sq

    def run(self):
        start = time.clock()
        count = 0
        self._logger.info("run start")
        with codecs.open(self.dst_file, 'w', 'utf-8') as w :
            for item in self.sq.naive().iterator():
                w.write(item.unicode_dumps() + "\n")
                count += 1

        self._logger.info("write count : %d" % count)

        end = time.clock()
        self._logger.info("%s file count %d, run time : %.03f seconds" % (self.dst_file, count, end - start))

    def stop(self):
            self.thread_stop = True<commit_msg>cleanup will be more proper<commit_after># -*- coding:utf-8 -*-
__author__ = 'nathan'

import threading
import logging
import time
import codecs

class BaseSync():

    def __init__(self):
        pass

    def setup(self):
        pass

    def partition(self):
        pass

    def merge(self):
        pass

    def target(self):
        pass

    def sync(self, target):
        pass

    def cleanup(self):
        pass

class Dumper(threading.Thread):
    def __init__(self, dst_file, sq):
        threading.Thread.__init__(self)
        self._logger = logging.getLogger("dbsync.syncs")
        self.dst_file = dst_file
        self.sq = sq

    def run(self):
        start = time.clock()
        count = 0
        self._logger.info("run start")
        with codecs.open(self.dst_file, 'w', 'utf-8') as w :
            for item in self.sq.naive().iterator():
                w.write(item.unicode_dumps() + "\n")
                count += 1

        self._logger.info("write count : %d" % count)

        end = time.clock()
        self._logger.info("%s file count %d, run time : %.03f seconds" % (self.dst_file, count, end - start))

    def stop(self):
            self.thread_stop = True
{'fraction_non_alphanumeric': 0.12442202606137033, 'fraction_numerical': 0.005044136191677175, 'mean_word_length': 1.9712858926342072, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6677300', 'n_tokens_mistral': 761, 'n_tokens_neox': 726, 'n_words': 189}