<commit_before># -*- coding=utf-8
from cos_auth import CosS3Auth
from cos_threadpool import SimpleThreadPool
import time
import requests
from os import path
from contextlib import closing
from xml.dom import minidom
import logging
import sys
import os
import base64

logger = logging.getLogger(__name__)
fs_coding = sys.getfilesystemencoding()


def to_unicode(s):
    if isinstance(s, unicode):
        return s
    else:
        return s.decode(fs_coding)


def to_printable_str(s):
    if isinstance(s, unicode):
        return s.encode(fs_coding)
    else:
        return s


def view_bar(num, total):
    ret = 1.0*num / total
    ag = ret * 100
    ab = "\r [%-50s]%.2f%%" % ('='*int(ret*50), ag, )
    sys.stdout.write(ab)
    sys.stdout.flush()


def getTagText(root, tag):
    node = root.getElementsByTagName(tag)[0]
    rc = ""
    for node in node.childNodes:
        if node.nodeType in (node.TEXT_NODE, node.CDATA_SECTION_NODE):
            rc = rc + node.data


def get_md5_filename(local_path, cos_path):
    ori_file = '~/.tmp/' + os.path.abspath(os.path.dirname(local_path)) + "!!!" + str(os.path.getsize(local_path)) + "!!!" + cos_path
    return base64.encodestring(os.path.expanduser(ori_file))[0:10]


def query_yes_no(question, default=None):
    valid = {"yes": True, "y": True, "ye": True,
             "no": False, "n": False}
    if default is None:
        prompt = " [y/n] "
    elif default == "yes":
        prompt = " [Y/n] "
    elif default == "no":
        prompt = " [y/N] "
    else:
        raise ValueError("invalid default answer: '%s'" % default)
    while True:
        sys.stdout.write(question + prompt)
        sys.stdout.flush()
        choice = raw_input().lower()
        if default is not None and choice == '':
            return valid[default]
        elif choice in valid:
            return valid[choice]
        else:
            sys.stdout.write("Please respond with 'yes' or 'no' "
                             "(or 'y' or 'n').\n")


class CosConfig(object):

    def __init__(self, appid, region, bucket, access_id, access_key, part_size=1, max_thread=5, *args, **kwargs):
        self._appid = appid
        self._region = region
        self._bucket = bucket
        self._access_id = access_id
        self._access_key = access_key
        self._part_size = min(10, part_size)
        self._max_thread = min(10, max_thread)
        logger.info("config parameter-> appid: {appid}, region: {region}, bucket: {bucket}, part_size: {part_size}, max_thread: {max_thread}".format(
                 appid=appid,
                 region=region,
                 bucket=bucket,
                 part_size=part_size,
                 max_thread=max_thread))

    def uri(self, path=None):
        if path:
            url = u"http://{bucket}-{uid}.{region}.myqcloud.com/{path}".format(
                bucket=self._bucket,
                uid=self._appid,
                region=self._region,
                path=to_unicode(path)
            )
        else:
            url = u"http://{bucket}-{uid}.{region}.myqcloud.com".format(
                bucket=self._bucket,
                uid=self._appid,
                region=self._region
            )
        return url


class ObjectInterface(object):

    def __init__(self, conf, session=None):
        self._conf = conf
        self._upload_id = None
        self._md5 = {}
        self._have_finished = 0
        self._err_tips = ''
        self._retry = 2
        self._file_num = 0
        self._folder_num = 0
        self._path_md5 = ""
        self._have_uploaded = []
        self._etag = 'ETag'
        if session is None:
            self._session = requests.session()
        else:
            self._session = session

    def list_part(self, cos_path):
        logger.info("getting uploaded parts")
        NextMarker = ""
        IsTruncated = "true"
        while IsTruncated == "true":
            url = self._conf.uri(path=cos_path+'?uploadId={UploadId}&upload&max-parts=1000&part-number-marker={nextmarker}'.format(UploadId=self._upload_id, nextmarker=NextMarker))
            rt = self._session.get(url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key))

            if rt.status_code == 200:
                root = minidom.parseString(rt.content).documentElement
                IsTruncated = root.getElementsByTagName("IsTruncated")[0].childNodes[0].data
                if IsTruncated == 'true':
                    NextMarker = root.getElementsByTagName("NextPartNumberMarker")[0].childNodes[0].data
                logger.debug("list resp, status code: {code}, headers: {headers}, text: {text}".format(
                     code=rt.status_code,
                     headers=rt.headers,
                     text=to_printable_str(rt.text)))
                contentset = root.getElementsByTagName("Part")
                for content in contentset:
                    ID = content.getElementsByTagName("PartNumber")[0].childNodes[0].data
                    self._have_uploaded.append(ID)
                    self._md5[int(ID)] = content.getElementsByTagName(self._etag)[0].childNodes[0].data[1:-1]
            else:
                logger.debug("list parts error")
                return False
        logger.debug("list parts error")
        return True

    def upload_folder(self, local_path, cos_path):

        local_path = to_unicode(local_path)
        cos_path = to_unicode(cos_path)
        filelist = os.listdir(local_path)
        if cos_path[-1] != '/':
            cos_path += '/'
        if local_path[-1] != '/':
            local_path += '/'
        self._folder_num += 1
        if len(filelist) == 0:
            logger.debug(cos_path+'tmp/')
            self.upload_file(local_path="", cos_path=cos_path+"tmp/")
        for filename in filelist:
            filepath = os.path.join(local_path, filename)<commit_msg>Update delete file count<commit_after># -*- coding=utf-8
from cos_auth import CosS3Auth
from cos_threadpool import SimpleThreadPool
import time
import requests
from os import path
from contextlib import closing
from xml.dom import minidom
import logging
import sys
import os
import base64

logger = logging.getLogger(__name__)
fs_coding = sys.getfilesystemencoding()


def to_unicode(s):
    if isinstance(s, unicode):
        return s
    else:
        return s.decode(fs_coding)


def to_printable_str(s):
    if isinstance(s, unicode):
        return s.encode(fs_coding)
    else:
        return s


def view_bar(num, total):
    ret = 1.0*num / total
    ag = ret * 100
    ab = "\r [%-50s]%.2f%%" % ('='*int(ret*50), ag, )
    sys.stdout.write(ab)
    sys.stdout.flush()


def getTagText(root, tag):
    node = root.getElementsByTagName(tag)[0]
    rc = ""
    for node in node.childNodes:
        if node.nodeType in (node.TEXT_NODE, node.CDATA_SECTION_NODE):
            rc = rc + node.data


def get_md5_filename(local_path, cos_path):
    ori_file = '~/.tmp/' + os.path.abspath(os.path.dirname(local_path)) + "!!!" + str(os.path.getsize(local_path)) + "!!!" + cos_path
    return base64.encodestring(os.path.expanduser(ori_file))[0:10]


def query_yes_no(question, default=None):
    valid = {"yes": True, "y": True, "ye": True,
             "no": False, "n": False}
    if default is None:
        prompt = " [y/n] "
    elif default == "yes":
        prompt = " [Y/n] "
    elif default == "no":
        prompt = " [y/N] "
    else:
        raise ValueError("invalid default answer: '%s'" % default)
    while True:
        sys.stdout.write(question + prompt)
        sys.stdout.flush()
        choice = raw_input().lower()
        if default is not None and choice == '':
            return valid[default]
        elif choice in valid:
            return valid[choice]
        else:
            sys.stdout.write("Please respond with 'yes' or 'no' "
                             "(or 'y' or 'n').\n")


class CosConfig(object):

    def __init__(self, appid, region, bucket, access_id, access_key, part_size=1, max_thread=5, *args, **kwargs):
        self._appid = appid
        self._region = region
        self._bucket = bucket
        self._access_id = access_id
        self._access_key = access_key
        self._part_size = min(10, part_size)
        self._max_thread = min(10, max_thread)
        logger.info("config parameter-> appid: {appid}, region: {region}, bucket: {bucket}, part_size: {part_size}, max_thread: {max_thread}".format(
                 appid=appid,
                 region=region,
                 bucket=bucket,
                 part_size=part_size,
                 max_thread=max_thread))

    def uri(self, path=None):
        if path:
            url = u"http://{bucket}-{uid}.{region}.myqcloud.com/{path}".format(
                bucket=self._bucket,
                uid=self._appid,
                region=self._region,
                path=to_unicode(path)
            )
        else:
            url = u"http://{bucket}-{uid}.{region}.myqcloud.com".format(
                bucket=self._bucket,
                uid=self._appid,
                region=self._region
            )
        return url


class ObjectInterface(object):

    def __init__(self, conf, session=None):
        self._conf = conf
        self._upload_id = None
        self._md5 = {}
        self._have_finished = 0
        self._err_tips = ''
        self._retry = 2
        self._file_num = 0
        self._folder_num = 0
        self._path_md5 = ""
        self._have_uploaded = []
        self._etag = 'ETag'
        if session is None:
            self._session = requests.session()
        else:
            self._session = session

    def list_part(self, cos_path):
        logger.info("getting uploaded parts")
        NextMarker = ""
        IsTruncated = "true"
        while IsTruncated == "true":
            url = self._conf.uri(path=cos_path+'?uploadId={UploadId}&upload&max-parts=1000&part-number-marker={nextmarker}'.format(UploadId=self._upload_id, nextmarker=NextMarker))
            rt = self._session.get(url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key))

            if rt.status_code == 200:
                root = minidom.parseString(rt.content).documentElement
                IsTruncated = root.getElementsByTagName("IsTruncated")[0].childNodes[0].data
                if IsTruncated == 'true':
                    NextMarker = root.getElementsByTagName("NextPartNumberMarker")[0].childNodes[0].data
                logger.debug("list resp, status code: {code}, headers: {headers}, text: {text}".format(
                     code=rt.status_code,
                     headers=rt.headers,
                     text=to_printable_str(rt.text)))
                contentset = root.getElementsByTagName("Part")
                for content in contentset:
                    ID = content.getElementsByTagName("PartNumber")[0].childNodes[0].data
                    self._have_uploaded.append(ID)
                    self._md5[int(ID)] = content.getElementsByTagName(self._etag)[0].childNodes[0].data[1:-1]
            else:
                logger.debug("list parts error")
                return False
        logger.debug("list parts error")
        return True

    def upload_folder(self, local_path, cos_path):

        local_path = to_unicode(local_path)
        cos_path = to_unicode(cos_path)
        filelist = os.listdir(local_path)
        if cos_path[-1] != '/':
            cos_path += '/'
        if local_path[-1] != '/':
            local_path += '/'
        self._folder_num += 1
        if len(filelist) == 0:
            self._file_num += 1
            logger.debug(cos_path+'tmp/')
            self.upload_file(local_path="", cos_path=cos_path+"tmp/")
        for filename in filelist:
            filepath = os.path.join(local_path, filename)
{'fraction_non_alphanumeric': 0.12418411460540815, 'fraction_numerical': 0.009578706450792574, 'mean_word_length': 2.0933403251179863, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2840258', 'n_tokens_mistral': 3702, 'n_tokens_neox': 3555, 'n_words': 736}