<commit_before>import sys
import numpy as np
from decimal import Decimal

def printlist(lst, name, **kw):
    mean = float(np.mean(lst,0))
    std = np.std(lst)
    stdstr = '%8.3f' % std if std < 10000 else '%8.5g' % std
    sovm = 100*std / mean if mean > 0 else float('nan')
    last = float(lst[-1])
    lstd = 100*(last - mean) / std if std > 0 else float('nan')
    print("{name:10s}={mean:10.5g}, σ={std:10.4g} ({sovm:9.3g}%) [{last:9.2f} ({lstd:7.3g}%)]".format(**locals()), **kw)

def to_dhms(tdelta):
    days = tdelta.days
    hr, sec = divmod(tdelta.seconds, 3600)
    mn, sec = divmod(sec, 60)
    return (days, hr, mn, sec)

def get_eta(cur, tot, starttime):
    from datetime import datetime, timedelta
    now = datetime.now()
    tused = now - starttime
    
    # timedeltas cannot be multiplied by floats (?), so we get out the 
    #seconds, use that, and make it back into a dt
    usedsecs = 24*60*60 * tused.days + tused.seconds + (1e-6 * tused.microseconds)
    tleft = (timedelta(0, (usedsecs) * ((tot - cur) / cur)) if cur > 0
            else timedelta(0, 0))
    
    endtime = now + tleft
    return endtime, to_dhms(tleft)

def interval_str(days, hr, mn, sec):
    return (('%dd ' % days) if days else '') + ('%d:%02d:%02d'% (hr, mn, sec))

def errprint(*args, **kw):
    kwargs = {'file':sys.stderr}
    kwargs.update(kw)
    print(*args, **kwargs)
    kwargs['file'].flush()

def filefinder(dir, *names, regexp=None, matchall=True, types=Decimal, ext = 'tsv.gz', **args):
    """
    dir             : Directory to look in
    names           : keywords to look for / store into
    regexp          : Define the regular expression for groups (default: #NAME##VALUE#-#NAME##VALUE#....ext)
    matchall        : Match the entire expression
    types           : What types to apply (default: all Decimal)
    ext             : File extension (ignores everything else) (default: 'tsv.gz')
    args            : Like names, but add some constants (such as T=1) for all fs
    """
    import re, fpath
    from namespace import Namespace

    try:
        iter(types)
    except TypeError:<commit_msg>added Progress class to util.py<commit_after>import sys
import numpy as np
from decimal import Decimal
from datetime import datetime, timedelta

def printlist(lst, name, **kw):
    mean = float(np.mean(lst,0))
    std = np.std(lst)
    stdstr = '%8.3f' % std if std < 10000 else '%8.5g' % std
    sovm = 100*std / mean if mean > 0 else float('nan')
    last = float(lst[-1])
    lstd = 100*(last - mean) / std if std > 0 else float('nan')
    print("{name:10s}={mean:10.5g}, σ={std:10.4g} ({sovm:9.3g}%) [{last:9.2f} ({lstd:7.3g}%)]".format(**locals()), **kw)

def to_dhms(tdelta):
    days = tdelta.days
    hr, sec = divmod(tdelta.seconds, 3600)
    mn, sec = divmod(sec, 60)
    return (days, hr, mn, sec)

def interval_str(days, hr, mn, sec):
    return (('%2dd ' % days) if days else '') + ('%2d:%02d:%02d'% (hr, mn, sec))
    
class Progress:
    def __init__(self, total):
        self.start = datetime.now()
        self.total = total
        
    def get_eta(self, i):
        """Returns (end time, time left) as datetime, timedelta objects."""
        now = datetime.now()
        tused = now - self.start
        
        # timedeltas cannot be multiplied by floats (?), so we get out the 
        #seconds, use that, and make it back into a dt
        usedsecs = 24*60*60 * tused.days + tused.seconds + (1e-6 * tused.microseconds)
        tleft = (timedelta(0, (usedsecs) * ((self.total - i) / i)) if i > 0
                else timedelta(0, 0))
        
        return now + tleft, tleft
    
    def eta_strs(self, i, timefmt = '(%a %b %d, %H:%M:%S)'):
        """Returns (time left, total time, endtime) as strs"""
        endtime, tleft = self.get_eta(i)
        interval = to_dhms(tleft)
        totinterval = to_dhms(endtime - self.start)
        
        return (interval_str(*interval), interval_str(*totinterval),
                    endtime.strftime(timefmt))
    
    def eta_str(self, i, timefmt = '(%a %b %d, %H:%M:%S)'):
        ss = self.eta_strs(i, timefmt=timefmt)
        return 'Ends in ({} / {}) on {}'.format(*ss)

def errprint(*args, **kw):
    kwargs = {'file':sys.stderr}
    kwargs.update(kw)
    print(*args, **kwargs)
    kwargs['file'].flush()

def filefinder(dir, *names, regexp=None, matchall=True, types=Decimal, ext = 'tsv.gz', **args):
    """
    dir             : Directory to look in
    names           : keywords to look for / store into
    regexp          : Define the regular expression for groups (default: #NAME##VALUE#-#NAME##VALUE#....ext)
    matchall        : Match the entire expression
    types           : What types to apply (default: all Decimal)
    ext             : File extension (ignores everything else) (default: 'tsv.gz')
    args            : Like names, but add some constants (such as T=1) for all fs
    """
    import re, fpath
    from namespace import Namespace

    try:
        iter(types)
    except TypeError:
{'fraction_non_alphanumeric': 0.14840637450199204, 'fraction_numerical': 0.022709163346613544, 'mean_word_length': 2.553432413305025, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2686594', 'n_tokens_mistral': 1787, 'n_tokens_neox': 1699, 'n_words': 518}