<commit_before># -*- coding:utf8 -*-
"""
Usage:
  rabbit.py imgetter <url> [<savedir>]
  rabbit.py splitfile <file_or_dir> [<size>]
  rabbit.py test
  rabbit.py urljoin <baseurl> [<url>]

Arguments:
  url           url
  savedir       dirpath
  size          int, ex:1 for 1M

Options:
  -h --help     show this help
"""
# todo: url2ip() to convert url to ip

import csv
import carrot
import datetime
import glob
import os
import re
import sys
import time
from decimal import Decimal
from docopt import docopt  # pip
from collections import defaultdict
# from hashlib import md5

reload(sys)
sys.setdefaultencoding('utf8')
BASE_DIR = os.path.split(os.path.realpath(__file__))[0]


# tree
tree = lambda: defaultdict(tree)


def filesplitter(*args, **kwargs):
    """
    split txtfile and return filenames
    @size=10: split file by 10M, default 10
    """
    input_filenames = []
    for arg in args:
        if os.path.isfile(arg):
            input_filenames.append(arg)
        elif os.path.isdir(arg):
            for parent, firnames, filenames in os.walk(arg):
                for filename in filenames:
                    input_filenames.append(os.path.
                                           join(os.path.
                                                abspath(parent),
                                                filename))
    try:
        size = kwargs.get('size')
        size = int(size) if size else 10
    except ValueError as e:
        raise ValueError('Invalid params: %s' % str(kwargs))
    _tmp_dir = filer(None, os.path.join(carrot.TMPDIR, 'data'))

    def _filewriter(filename, block):
        with open(sub_filename, 'w') as sub_f:
            sub_f.writelines(block)
        return True

    output_filenames = []
    for filename in input_filenames:
        sub_count = 0
        f = open(filename)
        while True:
            block = f.readlines(1024000 * size)  # 1M
            if not block:
                break
            sub_filename = os.path.join(_tmp_dir,  # tmp data dir
                                        ('-%s' % sub_count).  # subcount
                                        join(os.path.splitext(os.  # filename
                                        path.split(filename)[-1])))
            if _filewriter(filename, block):
                output_filenames.append(sub_filename)
            sub_count += 1
    return output_filenames


def xurljoin(base, url):
    """
    xurljoin(base, url)
    improved func for uelpsrse.urljoin
    article from: http://www.coder4.com/archives/2674
    @base   baseurl
    @url    path
    """
    from urlparse import urljoin
    from urlparse import urlparse
    from urlparse import urlunparse
    from posixpath import normpath

    url = url if url else ''
    url1 = urljoin(base, url)
    arr = urlparse(url1)
    path = normpath(arr[2])
    return urlunparse((arr.scheme, arr.netloc, path,
                       arr.params, arr.query, arr.fragment))


def imgetter(url, savedir=''):
    """
    download img from url, save to savedir
    @savedir path to save img

    usage:
    >>> url = 'http://www.itprofessor.cn/media/media.jpg'
    >>> imgetter(url)
    """
    from cStringIO import StringIO
    import purl  # pip
    import requests  # pip
    from PIL import Image  # pip
    url = url.strip()
    url_resp = requests.get(url)
    img = Image.open(StringIO(url_resp.content)) if url_resp.ok else None
    if isinstance(img, Image.Image):
        parse = purl.URL(url)
        imgname = os.path.splitext(parse.path().strip('/').
                                   replace('/', '_'))[0]
        savedir = savedir if savedir else ''
        img.save(filer('%s.%s' % (imgname, img.format.lower()),
                 savedir))
        return True
    else:
        raise TypeError('not a image url: %s' % url)


class PoolManager(object):
    from multiprocessing import cpu_count
    from multiprocessing import Pool

    def __init__(self, args):
        """
        PoolManager(args)

        usage:
        >>> jobs = [(func1, args1), (func2, args2)]
        >>> p = PoolManager(jobs)
        >>> p.run()
        """
        self.pool = Pool(cpu_count())
        self.args = [i for i in args]

    def add(self, args):
        self.args += [i for i in args]

    def run(self):
        self.collector = []
        jobs_num = len(self.args)
        bar_length = 30
        for index, (func, argv) in enumerate(self.args):
            if jobs_num > 1:
                percent = index / float(jobs_num - 1) * 100
                hashes = '#' * int(percent / 100.0 * bar_length)
                spaces = ' ' * (bar_length - len(hashes))
                sys.stdout.write("\r%s: [%s] %d%%" % ('Add',
                                                      hashes + spaces,
                                                      percent))
                sys.stdout.flush()
            result = self.pool.apply_async(func, argv)
            # result = self.pool.apply_async(func, (argv, ))
            self.collector.append([func.__name__, argv, result])
        self.pool.close()
        self.pool.join()

        # num of jobs in pool
        jobs_successful = 0
        while jobs_successful < jobs_num:
            for index, job in enumerate(self.collector):
                if jobs_num > 1:
                    percent = index / float(jobs_num - 1) * 100
                    hashes = '#' * int(percent / 100.0 * bar_length)
                    spaces = ' ' * (bar_length - len(hashes))
                    sys.stdout.write("\r%s: [%s] %d%%" % ('Run',
                                                          hashes + spaces,
                                                          percent))
                    sys.stdout.flush()

                result = job[-1]
                if not isinstance(result, list) and result.successful():
                    del self.collector[index][-1]
                    self.collector[index].append(result.get())
                    jobs_successful += 1
        sys.stdout.write('\n')
        sys.stdout.flush()
        return self.collector


def filer(filename='', filedir='./'):
    """
    filer(filename='', filedir='./')
    return fullpath
    auto create filedir
    if filename is null, only filedir be created
    """
    filedir = os.path.abspath(filedir)
    os.makedirs(filedir) if not os.path.isdir(filedir) else None
    return (os.path.join(filedir, filename) if filename else filedir)


def logger(name, logname, logdir='./'):
    """
    logger(name, logname, logdir='./')
    high level logging

    how to use:
    alog = logger('default', 'log13.txt')
    alog.debug('debug message!')
    alog.info('info message!')
    alog.error('error message')
    alog.critical('critical message')
    alog.warning('warning message')
    """
    import logging
    import logging.config
    logging.config.dictConfig({
        'version': 1,
        'disable_existing_loggers': True,
        'formatters': {
            'default': {'format': '%(asctime)s %(levelname)s %(message)s',
                        'datefmt': '%Y-%m-%d %H:%M:%S',
                        },
            'verbose': {'format': '%(asctime)s %(levelname)s %(module)s \
                %(process)d %(thread)d %(message)s',
                        'datefmt': '%Y-%m-%d %H:%M:%S',
                        },
            'simple': {'format': '%(levelname)s %(message)s',
                       'datefmt': '%Y-%m-%d %H:%M:%S'
                       },
        },
        'handlers': {
            'console': {
                'level': 'DEBUG',
                'class': 'logging.StreamHandler',
                'formatter': 'default',
                'stream': 'ext://sys.stdout',
            },
            'file': {
                'level': 'INFO',
                'class': 'logging.handlers.RotatingFileHandler',
                'formatter': 'default',
                'filename': filer(logname, logdir),
                'maxBytes': 1024000,
                'backupCount': 3,
                'encoding': 'utf8',
            },
        },
        'loggers': {
            'default': {
                'level': 'DEBUG',
                'handlers': ['console', 'file'],
            },
            'file': {
                'level': 'INFO',
                'handlers': ['file'],
            }
        },
        # 'disable_existing_loggers': False,
    })
    return logging.getLogger(name)


def func_monitor(trace_this=True):
    """
    func_monitor(trace_this=True)
    A decorator to show running time of a func
    @trace_this: print running_time or not

    eg:
    @func_monitor(trace_this=True)
    def func...
    """
    from functools import wraps
    if trace_this:
        def _time_decorator(func):
            @wraps(func)
            def _wrapper(*args, **kwargs):
                _start_time = datetime.datetime.now()
                _resp = func(*args, **kwargs)
                _end_time = datetime.datetime.now()
                _time_diff = (_end_time - _start_time).total_seconds()
                print (u"<%s> used %s s." % (func.__name__, _time_diff))
                return _resp
            return _wrapper
    else:
        @wraps(func)
        def _time_decorator(func):
            return func
    return _time_decorator


def time_builder(basetime='', timedelta=0, target_type='day'):
    """
    time_builder(basetime='', timedelta=0, target_type='day')
    Generate time with target_type
    @basetime: input a time or None by default
    @timedelta: basetime + timedelta = target_time
    @target_type: day, month, year, month_start

    # doctest
    >>> time_builder(basetime='20160310')
    '2016-03-10'
    >>> time_builder(basetime='20160310', target_type='month_start')
    '2016-03-01'
    >>> time_builder(basetime='20160310', target_type='year')
    '2016'
    >>> time_builder(basetime='20160310', timedelta=-1, target_type='day')
    '2016-03-09'
    >>> time_builder(basetime='20160310', timedelta=30, target_type='month')
    '2016-04'
    """
    if basetime:
        basetime = unicode(basetime).replace("-", "")
        try:
            basetime = time.strptime(basetime, '%Y%m%d')
            basetime = datetime.datetime(*basetime[:3])
        except:
            raise ValueError(r'basetime should be None or an exsit day')
            # return False
    else:
        basetime = datetime.datetime.now()

    _oneday = datetime.timedelta(days=1)
    _target_type_dict = {
        'year': '%Y',
        'month': '%Y-%m',
        'month_start': '%Y-%m-01',
        'day': '%Y-%m-%d',
        'hour': '%Y-%m-%d %H:00:00',
        'minute': '%Y-%m-%d %H:%M:00',
        'second': '%Y-%m-%d %H:%M:%S',
        'dbYHMS/': '%d/%b/%Y:%H:%M:%S',
        'dbY/': '%d/%b/%Y',  # '27/Mar/2016'
        'dbY-': '%d-%b-%Y',  # '27-Mar-2016'
        'mdYHMS/': '%m/%d/%Y:%H:%M:%S',
        'mdY000/': '%m/%d/%Y:00:00:00',
        'full_time': "%a, %d %b %Y %H:%M:%S %z",
    }
    _target_time = basetime + _oneday * timedelta
    try:
        return datetime.datetime.strftime(_target_time,
                                          _target_type_dict[target_type])
    except KeyError as e:
        raise KeyError("`target_type` must in %s" % _target_type_dict.keys())


def lister(*args, **kwargs):
    """
    lister(*args, **kwargs)
    Convert int, tuple, etc to list with target type if it can be
    @kwargs['target_type']<var_type>: int, str(unicode), unicode,
                                      float, Deciaml

    # doctest
    >>> lister(1, 2, 3)
    [1, 2, 3]
    >>> lister(1, 2, 3, target_type=float)
    [1.0, 2.0, 3.0]
    >>> lister(1, 2, 3, target_type=str)
    [u'1', u'2', u'3']
    >>> lister('a', 'b', '3', target_type=int)
    ['a', 'b', 3]
    >>> lister(1, 2, (3, 4), [[[5], 6], 7], )
    [1, 2, 3, 4, 5, 6, 7]
    """
    target_type = kwargs['target_type'] if kwargs else None
    target_type = unicode if target_type in (str,) else target_type
    _collector = []

    def _iterator(items, target_type):
        for item in items:
            if isinstance(item, (list, tuple, set,)):
                _iterator(item, target_type)
            else:
                try:
                    _collector.append(target_type(item))
                except (ValueError, TypeError) as e:
                    _collector.append(item)
    _iterator(args, target_type)
    return _collector


def distinct(*args):
    """
    distinct(*args)
    return distinct items
    """
    items = lister(args)
    func = lambda x, y: x if y in x else x + [y]
    return reduce(func, [[], ] + items)


def xls2dict(filename):
    """
    xls2dict(filename)
    get a xls or xlsx, return a list
    """
    import xlrd  # pip

    collector = []
    _full_filename = os.path.abspath(filename)
    try:
        data = xlrd.open_workbook(_full_filename)
    except:
        raise ValueError('%s is not a xls_type file' % filename)

    for table in data.sheets():
        table_collector = []
        nrows = table.nrows
        ncols = table.ncols
        if not (nrows and ncols):
            continue
        for row in range(nrows):
            table_collector.append(table.row_values(row))
        table_info = {
            'name': table.name,
            'data': table_collector,
            'rows': nrows,
            'cols': ncols,
            'title': table.row_values(0)
        }
        collector.append(table_info)
    return collector


def csv2xls(filename):
    """
    csv2xls(filename)
    given a csv file, convert to xls and save at the same dir
    warning: maxline_of_csvfile <=65535
    """
    _full_filename = os.path.abspath(filename)
    try:
        _csv_reader = csv.reader(open(_full_filename))
    except:
        raise ValueError('%s is not a csv_type file' % filename)
    _xls_name = re.sub('\..*$', '.xls', _full_filename, 1)
    _xlsapp = XlsManager(_xls_name)
    for row in _csv_reader:
        _xlsapp.writerow(row)

    _xlsapp.close()
    return True


def imager(url, **kwargs):
    """
    default url
    """
    pass


class CsvManager(object):
    def __init__(self, filename, mode='wb', filedir='./', **kwargs):
        """
        CsvManager(filename, mode='wb', filedir='./')
        <class>: wrapper csv model, adapt to Chinese
        @filename<str>: file name
        @mode<str>: open_mode
        @filedir<str>: filedir to save file
        todo: addtitle
        """
        self.title = kwargs.get('title', None)

        # makedirs if not exsit
        filedir = os.path.abspath(filedir)
        os.makedirs(filedir) if not os.path.isdir(filedir) else True
        # create csv file<commit_msg>delete imager<commit_after># -*- coding:utf8 -*-
"""
Usage:
  rabbit.py imgetter <url> [<savedir>]
  rabbit.py splitfile <file_or_dir> [<size>]
  rabbit.py test
  rabbit.py urljoin <baseurl> [<url>]

Arguments:
  url           url
  savedir       dirpath
  size          int, ex:1 for 1M

Options:
  -h --help     show this help
"""
# todo: url2ip() to convert url to ip

import csv
import carrot
import datetime
import glob
import os
import re
import sys
import time
from decimal import Decimal
from docopt import docopt  # pip
from collections import defaultdict
# from hashlib import md5

reload(sys)
sys.setdefaultencoding('utf8')
BASE_DIR = os.path.split(os.path.realpath(__file__))[0]


# tree
tree = lambda: defaultdict(tree)


def filesplitter(*args, **kwargs):
    """
    split txtfile and return filenames
    @size=10: split file by 10M, default 10
    """
    input_filenames = []
    for arg in args:
        if os.path.isfile(arg):
            input_filenames.append(arg)
        elif os.path.isdir(arg):
            for parent, firnames, filenames in os.walk(arg):
                for filename in filenames:
                    input_filenames.append(os.path.
                                           join(os.path.
                                                abspath(parent),
                                                filename))
    try:
        size = kwargs.get('size')
        size = int(size) if size else 10
    except ValueError as e:
        raise ValueError('Invalid params: %s' % str(kwargs))
    _tmp_dir = filer(None, os.path.join(carrot.TMPDIR, 'data'))

    def _filewriter(filename, block):
        with open(sub_filename, 'w') as sub_f:
            sub_f.writelines(block)
        return True

    output_filenames = []
    for filename in input_filenames:
        sub_count = 0
        f = open(filename)
        while True:
            block = f.readlines(1024000 * size)  # 1M
            if not block:
                break
            sub_filename = os.path.join(_tmp_dir,  # tmp data dir
                                        ('-%s' % sub_count).  # subcount
                                        join(os.path.splitext(os.  # filename
                                        path.split(filename)[-1])))
            if _filewriter(filename, block):
                output_filenames.append(sub_filename)
            sub_count += 1
    return output_filenames


def xurljoin(base, url):
    """
    xurljoin(base, url)
    improved func for uelpsrse.urljoin
    article from: http://www.coder4.com/archives/2674
    @base   baseurl
    @url    path
    """
    from urlparse import urljoin
    from urlparse import urlparse
    from urlparse import urlunparse
    from posixpath import normpath

    url = url if url else ''
    url1 = urljoin(base, url)
    arr = urlparse(url1)
    path = normpath(arr[2])
    return urlunparse((arr.scheme, arr.netloc, path,
                       arr.params, arr.query, arr.fragment))


def imgetter(url, savedir=''):
    """
    download img from url, save to savedir
    @savedir path to save img

    usage:
    >>> url = 'http://www.itprofessor.cn/media/media.jpg'
    >>> imgetter(url)
    """
    from cStringIO import StringIO
    import purl  # pip
    import requests  # pip
    from PIL import Image  # pip
    url = url.strip()
    url_resp = requests.get(url)
    img = Image.open(StringIO(url_resp.content)) if url_resp.ok else None
    if isinstance(img, Image.Image):
        parse = purl.URL(url)
        imgname = os.path.splitext(parse.path().strip('/').
                                   replace('/', '_'))[0]
        savedir = savedir if savedir else ''
        img.save(filer('%s.%s' % (imgname, img.format.lower()),
                 savedir))
        return True
    else:
        raise TypeError('not a image url: %s' % url)


class PoolManager(object):
    from multiprocessing import cpu_count
    from multiprocessing import Pool

    def __init__(self, args):
        """
        PoolManager(args)

        usage:
        >>> jobs = [(func1, args1), (func2, args2)]
        >>> p = PoolManager(jobs)
        >>> p.run()
        """
        self.pool = Pool(cpu_count())
        self.args = [i for i in args]

    def add(self, args):
        self.args += [i for i in args]

    def run(self):
        self.collector = []
        jobs_num = len(self.args)
        bar_length = 30
        for index, (func, argv) in enumerate(self.args):
            if jobs_num > 1:
                percent = index / float(jobs_num - 1) * 100
                hashes = '#' * int(percent / 100.0 * bar_length)
                spaces = ' ' * (bar_length - len(hashes))
                sys.stdout.write("\r%s: [%s] %d%%" % ('Add',
                                                      hashes + spaces,
                                                      percent))
                sys.stdout.flush()
            result = self.pool.apply_async(func, argv)
            # result = self.pool.apply_async(func, (argv, ))
            self.collector.append([func.__name__, argv, result])
        self.pool.close()
        self.pool.join()

        # num of jobs in pool
        jobs_successful = 0
        while jobs_successful < jobs_num:
            for index, job in enumerate(self.collector):
                if jobs_num > 1:
                    percent = index / float(jobs_num - 1) * 100
                    hashes = '#' * int(percent / 100.0 * bar_length)
                    spaces = ' ' * (bar_length - len(hashes))
                    sys.stdout.write("\r%s: [%s] %d%%" % ('Run',
                                                          hashes + spaces,
                                                          percent))
                    sys.stdout.flush()

                result = job[-1]
                if not isinstance(result, list) and result.successful():
                    del self.collector[index][-1]
                    self.collector[index].append(result.get())
                    jobs_successful += 1
        sys.stdout.write('\n')
        sys.stdout.flush()
        return self.collector


def filer(filename='', filedir='./'):
    """
    filer(filename='', filedir='./')
    return fullpath
    auto create filedir
    if filename is null, only filedir be created
    """
    filedir = os.path.abspath(filedir)
    os.makedirs(filedir) if not os.path.isdir(filedir) else None
    return (os.path.join(filedir, filename) if filename else filedir)


def logger(name, logname, logdir='./'):
    """
    logger(name, logname, logdir='./')
    high level logging

    how to use:
    alog = logger('default', 'log13.txt')
    alog.debug('debug message!')
    alog.info('info message!')
    alog.error('error message')
    alog.critical('critical message')
    alog.warning('warning message')
    """
    import logging
    import logging.config
    logging.config.dictConfig({
        'version': 1,
        'disable_existing_loggers': True,
        'formatters': {
            'default': {'format': '%(asctime)s %(levelname)s %(message)s',
                        'datefmt': '%Y-%m-%d %H:%M:%S',
                        },
            'verbose': {'format': '%(asctime)s %(levelname)s %(module)s \
                %(process)d %(thread)d %(message)s',
                        'datefmt': '%Y-%m-%d %H:%M:%S',
                        },
            'simple': {'format': '%(levelname)s %(message)s',
                       'datefmt': '%Y-%m-%d %H:%M:%S'
                       },
        },
        'handlers': {
            'console': {
                'level': 'DEBUG',
                'class': 'logging.StreamHandler',
                'formatter': 'default',
                'stream': 'ext://sys.stdout',
            },
            'file': {
                'level': 'INFO',
                'class': 'logging.handlers.RotatingFileHandler',
                'formatter': 'default',
                'filename': filer(logname, logdir),
                'maxBytes': 1024000,
                'backupCount': 3,
                'encoding': 'utf8',
            },
        },
        'loggers': {
            'default': {
                'level': 'DEBUG',
                'handlers': ['console', 'file'],
            },
            'file': {
                'level': 'INFO',
                'handlers': ['file'],
            }
        },
        # 'disable_existing_loggers': False,
    })
    return logging.getLogger(name)


def func_monitor(trace_this=True):
    """
    func_monitor(trace_this=True)
    A decorator to show running time of a func
    @trace_this: print running_time or not

    eg:
    @func_monitor(trace_this=True)
    def func...
    """
    from functools import wraps
    if trace_this:
        def _time_decorator(func):
            @wraps(func)
            def _wrapper(*args, **kwargs):
                _start_time = datetime.datetime.now()
                _resp = func(*args, **kwargs)
                _end_time = datetime.datetime.now()
                _time_diff = (_end_time - _start_time).total_seconds()
                print (u"<%s> used %s s." % (func.__name__, _time_diff))
                return _resp
            return _wrapper
    else:
        @wraps(func)
        def _time_decorator(func):
            return func
    return _time_decorator


def time_builder(basetime='', timedelta=0, target_type='day'):
    """
    time_builder(basetime='', timedelta=0, target_type='day')
    Generate time with target_type
    @basetime: input a time or None by default
    @timedelta: basetime + timedelta = target_time
    @target_type: day, month, year, month_start

    # doctest
    >>> time_builder(basetime='20160310')
    '2016-03-10'
    >>> time_builder(basetime='20160310', target_type='month_start')
    '2016-03-01'
    >>> time_builder(basetime='20160310', target_type='year')
    '2016'
    >>> time_builder(basetime='20160310', timedelta=-1, target_type='day')
    '2016-03-09'
    >>> time_builder(basetime='20160310', timedelta=30, target_type='month')
    '2016-04'
    """
    if basetime:
        basetime = unicode(basetime).replace("-", "")
        try:
            basetime = time.strptime(basetime, '%Y%m%d')
            basetime = datetime.datetime(*basetime[:3])
        except:
            raise ValueError(r'basetime should be None or an exsit day')
            # return False
    else:
        basetime = datetime.datetime.now()

    _oneday = datetime.timedelta(days=1)
    _target_type_dict = {
        'year': '%Y',
        'month': '%Y-%m',
        'month_start': '%Y-%m-01',
        'day': '%Y-%m-%d',
        'hour': '%Y-%m-%d %H:00:00',
        'minute': '%Y-%m-%d %H:%M:00',
        'second': '%Y-%m-%d %H:%M:%S',
        'dbYHMS/': '%d/%b/%Y:%H:%M:%S',
        'dbY/': '%d/%b/%Y',  # '27/Mar/2016'
        'dbY-': '%d-%b-%Y',  # '27-Mar-2016'
        'mdYHMS/': '%m/%d/%Y:%H:%M:%S',
        'mdY000/': '%m/%d/%Y:00:00:00',
        'full_time': "%a, %d %b %Y %H:%M:%S %z",
    }
    _target_time = basetime + _oneday * timedelta
    try:
        return datetime.datetime.strftime(_target_time,
                                          _target_type_dict[target_type])
    except KeyError as e:
        raise KeyError("`target_type` must in %s" % _target_type_dict.keys())


def lister(*args, **kwargs):
    """
    lister(*args, **kwargs)
    Convert int, tuple, etc to list with target type if it can be
    @kwargs['target_type']<var_type>: int, str(unicode), unicode,
                                      float, Deciaml

    # doctest
    >>> lister(1, 2, 3)
    [1, 2, 3]
    >>> lister(1, 2, 3, target_type=float)
    [1.0, 2.0, 3.0]
    >>> lister(1, 2, 3, target_type=str)
    [u'1', u'2', u'3']
    >>> lister('a', 'b', '3', target_type=int)
    ['a', 'b', 3]
    >>> lister(1, 2, (3, 4), [[[5], 6], 7], )
    [1, 2, 3, 4, 5, 6, 7]
    """
    target_type = kwargs['target_type'] if kwargs else None
    target_type = unicode if target_type in (str,) else target_type
    _collector = []

    def _iterator(items, target_type):
        for item in items:
            if isinstance(item, (list, tuple, set,)):
                _iterator(item, target_type)
            else:
                try:
                    _collector.append(target_type(item))
                except (ValueError, TypeError) as e:
                    _collector.append(item)
    _iterator(args, target_type)
    return _collector


def distinct(*args):
    """
    distinct(*args)
    return distinct items
    same func as set
    """
    items = lister(args)
    func = lambda x, y: x if y in x else x + [y]
    return reduce(func, [[], ] + items)


def xls2dict(filename):
    """
    xls2dict(filename)
    get a xls or xlsx, return a list
    """
    import xlrd  # pip

    collector = []
    _full_filename = os.path.abspath(filename)
    try:
        data = xlrd.open_workbook(_full_filename)
    except:
        raise ValueError('%s is not a xls_type file' % filename)

    for table in data.sheets():
        table_collector = []
        nrows = table.nrows
        ncols = table.ncols
        if not (nrows and ncols):
            continue
        for row in range(nrows):
            table_collector.append(table.row_values(row))
        table_info = {
            'name': table.name,
            'data': table_collector,
            'rows': nrows,
            'cols': ncols,
            'title': table.row_values(0)
        }
        collector.append(table_info)
    return collector


def csv2xls(filename):
    """
    csv2xls(filename)
    given a csv file, convert to xls and save at the same dir
    warning: maxline_of_csvfile <=65535
    """
    _full_filename = os.path.abspath(filename)
    try:
        _csv_reader = csv.reader(open(_full_filename))
    except:
        raise ValueError('%s is not a csv_type file' % filename)
    _xls_name = re.sub('\..*$', '.xls', _full_filename, 1)
    _xlsapp = XlsManager(_xls_name)
    for row in _csv_reader:
        _xlsapp.writerow(row)

    _xlsapp.close()
    return True


class CsvManager(object):
    def __init__(self, filename, mode='wb', filedir='./', **kwargs):
        """
        CsvManager(filename, mode='wb', filedir='./')
        <class>: wrapper csv model, adapt to Chinese
        @filename<str>: file name
        @mode<str>: open_mode
        @filedir<str>: filedir to save file
        todo: addtitle
        """
        self.title = kwargs.get('title', None)

        # makedirs if not exsit
        filedir = os.path.abspath(filedir)
        os.makedirs(filedir) if not os.path.isdir(filedir) else True
        # create csv file
{'fraction_non_alphanumeric': 0.13441578460589867, 'fraction_numerical': 0.01596273079162813, 'mean_word_length': 1.810899287502407, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 121, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 44, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2907812', 'n_tokens_mistral': 9442, 'n_tokens_neox': 8934, 'n_words': 2154}