<commit_before>import datetime
import os
import random
import signal
import six
import sys
import time
import pytz
import dateutil.parser

from patroni.exceptions import PatroniException

__ignore_sigterm = False
__interrupted_sleep = False
__reap_children = False


def calculate_ttl(expiration):
    """
    >>> calculate_ttl(None)
    >>> calculate_ttl('2015-06-10 12:56:30.552539016Z') < 0
    True
    >>> calculate_ttl('2015-06-10T12:56:30.552539016Z') < 0
    True
    >>> calculate_ttl('fail-06-10T12:56:30.552539016Z')
    """
    if not expiration:
        return None
    try:
        expiration = dateutil.parser.parse(expiration)
    except (ValueError, TypeError):
        return None
    now = datetime.datetime.now(pytz.utc)
    return int((expiration - now).total_seconds())


def deep_compare(obj1, obj2):
    """
    >>> deep_compare({'1': None}, {})
    False
    >>> deep_compare({'1': {}}, {'1': None})
    False
    >>> deep_compare({'1': [1]}, {'1': [2]})
    False
    >>> deep_compare({'1': 2}, {'1': '2'})
    True
    >>> deep_compare({'1': {'2': [3, 4]}}, {'1': {'2': [3, 4]}})
    True
    """

    if set(list(obj1.keys())) != set(list(obj2.keys())):  # Objects have different sets of keys
        return False

    for key, value in obj1.items():
        if isinstance(value, dict):
            if not (isinstance(obj2[key], dict) and deep_compare(value, obj2[key])):
                return False
        elif str(value) != str(obj2[key]):
            return False
    return True


def parse_bool(value):
    """
    >>> parse_bool(1)
    True
    >>> parse_bool('off')
    False
    >>> parse_bool('foo')
    """
    value = str(value).lower()
    if value in ('on', 'true', 'yes', '1'):
        return True
    if value in ('off', 'false', 'no', '0'):
        return False


def strtol(value, strict=True):
    """As most as possible close equivalent of strtol(3) function (with base=0),
       used by postgres to parse parameter values.
    >>> strtol(1) == (1, '')
    True
    >>> strtol(' +0x400MB') == (1024, 'MB')
    True
    >>> strtol(' -070d') == (-56, 'd')
    True
    >>> strtol(' d ') == (None, 'd')
    True
    >>> strtol(' s ', False) == (1, 's')
    True
    """
    value = str(value).strip()
    l = len(value)
    i = 0
    # skip sign:
    if i < l and value[i] in ('-', '+'):
        i += 1

    # we always expect to get digit in the beginning
    if i < l and value[i].isdigit():
        if value[i] == '0':
            i += 1
            if i < l and value[i] == 'x':  # '0' followed by 'x': HEX
                base = 16
                i += 1
            else:  # just starts with '0': OCT
                base = 8
        else:  # any other digit: DEC
            base = 10

        ret = None
        while i < l:
            try:  # try to find maximally long number
                i += 1  # by giving to `int` longer and longer strings
                ret = int(value[:i], base) if six.PY3 else long(value[:i], base)
            except ValueError:  # until we will not get an exception or end of the string
                i -= 1
                break
        if ret is not None:  # yay! there is a number in the beginning of the string
            return ret, value[i:].strip()  # return the number and the "rest"

    return (None if strict else 1), value.strip()


def parse_int(value, base_unit=None):
    """
    >>> parse_int('1') == 1
    True
    >>> parse_int(' 0x400 MB ', '16384kB') == 64
    True
    >>> parse_int('1MB', 'kB') == 1024
    True
    >>> parse_int('1000 ms', 's') == 1
    True<commit_msg>Hexadecimal can be written as 0X<commit_after>import datetime
import os
import random
import signal
import six
import sys
import time
import pytz
import dateutil.parser

from patroni.exceptions import PatroniException

__ignore_sigterm = False
__interrupted_sleep = False
__reap_children = False


def calculate_ttl(expiration):
    """
    >>> calculate_ttl(None)
    >>> calculate_ttl('2015-06-10 12:56:30.552539016Z') < 0
    True
    >>> calculate_ttl('2015-06-10T12:56:30.552539016Z') < 0
    True
    >>> calculate_ttl('fail-06-10T12:56:30.552539016Z')
    """
    if not expiration:
        return None
    try:
        expiration = dateutil.parser.parse(expiration)
    except (ValueError, TypeError):
        return None
    now = datetime.datetime.now(pytz.utc)
    return int((expiration - now).total_seconds())


def deep_compare(obj1, obj2):
    """
    >>> deep_compare({'1': None}, {})
    False
    >>> deep_compare({'1': {}}, {'1': None})
    False
    >>> deep_compare({'1': [1]}, {'1': [2]})
    False
    >>> deep_compare({'1': 2}, {'1': '2'})
    True
    >>> deep_compare({'1': {'2': [3, 4]}}, {'1': {'2': [3, 4]}})
    True
    """

    if set(list(obj1.keys())) != set(list(obj2.keys())):  # Objects have different sets of keys
        return False

    for key, value in obj1.items():
        if isinstance(value, dict):
            if not (isinstance(obj2[key], dict) and deep_compare(value, obj2[key])):
                return False
        elif str(value) != str(obj2[key]):
            return False
    return True


def parse_bool(value):
    """
    >>> parse_bool(1)
    True
    >>> parse_bool('off')
    False
    >>> parse_bool('foo')
    """
    value = str(value).lower()
    if value in ('on', 'true', 'yes', '1'):
        return True
    if value in ('off', 'false', 'no', '0'):
        return False


def strtol(value, strict=True):
    """As most as possible close equivalent of strtol(3) function (with base=0),
       used by postgres to parse parameter values.
    >>> strtol(1) == (1, '')
    True
    >>> strtol(' +0x400MB') == (1024, 'MB')
    True
    >>> strtol(' -070d') == (-56, 'd')
    True
    >>> strtol(' d ') == (None, 'd')
    True
    >>> strtol(' s ', False) == (1, 's')
    True
    """
    value = str(value).strip()
    l = len(value)
    i = 0
    # skip sign:
    if i < l and value[i] in ('-', '+'):
        i += 1

    # we always expect to get digit in the beginning
    if i < l and value[i].isdigit():
        if value[i] == '0':
            i += 1
            if i < l and value[i] in ('x', 'X'):  # '0' followed by 'x': HEX
                base = 16
                i += 1
            else:  # just starts with '0': OCT
                base = 8
        else:  # any other digit: DEC
            base = 10

        ret = None
        while i < l:
            try:  # try to find maximally long number
                i += 1  # by giving to `int` longer and longer strings
                ret = int(value[:i], base) if six.PY3 else long(value[:i], base)
            except ValueError:  # until we will not get an exception or end of the string
                i -= 1
                break
        if ret is not None:  # yay! there is a number in the beginning of the string
            return ret, value[i:].strip()  # return the number and the "rest"

    return (None if strict else 1), value.strip()


def parse_int(value, base_unit=None):
    """
    >>> parse_int('1') == 1
    True
    >>> parse_int(' 0x400 MB ', '16384kB') == 64
    True
    >>> parse_int('1MB', 'kB') == 1024
    True
    >>> parse_int('1000 ms', 's') == 1
    True
{'fraction_non_alphanumeric': 0.15828237909949971, 'fraction_numerical': 0.04294052251250695, 'mean_word_length': 2.188746123172353, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 129, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4691913', 'n_tokens_mistral': 2550, 'n_tokens_neox': 2344, 'n_words': 722}