<commit_before>"""
This module provides some useful tools for ``vcs`` like annotate/diff html
output. It also includes some internal helpers.
"""
import time
import datetime


def makedate():
    lt = time.localtime()
    if lt[8] == 1 and time.daylight:
        tz = time.altzone
    else:
        tz = time.timezone
    return time.mktime(lt), tz


def date_fromtimestamp(unixts, tzoffset=0):
    """
    Makes a datetime objec out of unix timestamp with given timezone offset
    :param unixts:
    :param tzoffset:
    """
    return datetime.datetime(*time.gmtime(float(unixts) - tzoffset)[:6])


def safe_unicode(str):
    """
    safe unicode function. In case of UnicodeDecode error we try to return
    unicode with errors replace, if this failes we return unicode with
    string_escape decoding
    """

    try:
        u_str = unicode(str)
    except UnicodeDecodeError:
        try:
            u_str = unicode(str, 'utf-8', 'replace')
        except UnicodeDecodeError:
            # In case we have a decode error just represent as byte string
            u_str = unicode(str(str).encode('string_escape'))

    return u_str<commit_msg>param should be not named as function name<commit_after>"""
This module provides some useful tools for ``vcs`` like annotate/diff html
output. It also includes some internal helpers.
"""
import time
import datetime


def makedate():
    lt = time.localtime()
    if lt[8] == 1 and time.daylight:
        tz = time.altzone
    else:
        tz = time.timezone
    return time.mktime(lt), tz


def date_fromtimestamp(unixts, tzoffset=0):
    """
    Makes a datetime objec out of unix timestamp with given timezone offset
    :param unixts:
    :param tzoffset:
    """
    return datetime.datetime(*time.gmtime(float(unixts) - tzoffset)[:6])


def safe_unicode(s):
    """
    safe unicode function. In case of UnicodeDecode error we try to return
    unicode with errors replace, if this fails we return unicode with
    string_escape decoding
    """

    try:
        u_str = unicode(s)
    except UnicodeDecodeError:
        try:
            u_str = unicode(s, 'utf-8', 'replace')
        except UnicodeDecodeError:
            # In case we have a decode error just represent as byte string
            u_str = unicode(str(s).encode('string_escape'))

    return u_str
{'fraction_non_alphanumeric': 0.09043927648578812, 'fraction_numerical': 0.004306632213608958, 'mean_word_length': 2.820723684210526, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1033808', 'n_tokens_mistral': 723, 'n_tokens_neox': 664, 'n_words': 240}