<commit_before># -*- coding: utf-8 -*-
"""
Common utility functions for intelmq.

decode
encode
base64_decode
base64_encode
load_configuration
log
reverse_readline
parse_logline
"""
import base64
import json
import logging
import logging.handlers
import os
import re
import sys
import traceback

import pkg_resources

import intelmq

__all__ = ['base64_decode', 'base64_encode', 'decode', 'encode',
           'load_configuration', 'load_parameters', 'log', 'parse_logline',
           'reverse_readline', 'error_message_from_exc',
           ]

# Used loglines format
LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
LOG_FORMAT_STREAM = '%(name)s: %(message)s'

# Regex for parsing the above LOG_FORMAT
LOG_REGEX = (r'^(?P<date>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d+) -'
             r' (?P<bot_id>[-\w]+) - '
             r'(?P<log_level>[A-Z]+) - '
             r'(?P<message>.+)$')


class Parameters(object):
    pass


def decode(text, encodings=("utf-8", ), force=False):
    """
    Decode given string to UTF-8 (default).

    Parameters:
    -----------
    text : bytes string
        if unicode string is given, same object is returned
    encodings : iterable of strings
        list/tuple of encodings to use, default ('utf-8')
    force : boolean
        Ignore invalid characters, default: False

    Returns
    -------
    text : unicode string
        unicode string is always returned, even when encoding is ascii
        (Python 3 compat)
    """
    if isinstance(text, str):
        return text

    for encoding in encodings:
        try:
            return str(text.decode(encoding))
        except ValueError:
            pass

    if force:
        for encoding in encodings:
            try:
                return str(text.decode(encoding, 'ignore'))
            except ValueError:
                pass

    raise ValueError("Could not decode string with given encodings{!r}"
                     ".".format(encodings))


def encode(text, encodings=("utf-8", ), force=False):
    """
    Encode given string from UTF-8 (default).

    Parameters:
    -----------
    text : unicode string
        if bytes string is given, same object is returned
    encodings : iterable of strings
        list/tuple of encodings to use, default ('utf-8')
    force : boolean
        Ignore invalid characters, default: False
    """
    if isinstance(text, bytes):
        return text

    for encoding in encodings:
        try:
            return text.encode(encoding)
        except ValueError:
            pass

    if force:
        for encoding in encodings:
            try:
                return text.encode(encoding, 'ignore')
            except ValueError:
                pass

    raise ValueError("Could not encode string with given encodings{!r}"
                     ".".format(encodings))


def base64_decode(value):
    """
    Parameters
    ----------
    value : string
        base 64, will be encoded to bytes if not already.

    Returns
    -------
    retval : unicode string

    Notes
    -----
    Possible bytes - unicode conversions problems are ignored.
    """
    return decode(base64.b64decode(encode(value, force=True)), force=True)


def base64_encode(value):
    """
    Parameters
    ----------
    value : string
        Will be encoded to bytes if not already of type bytes.

    Returns
    -------
    retval : unicode string

    Notes
    -----
    Possible bytes - unicode conversions problems are ignored.
    """
    return decode(base64.b64encode(encode(value, force=True)), force=True)


def load_configuration(configuration_filepath):
    """
    Load JSON configuration file.

    Parameters:
    -----------
    configuration_filepath : string
        Path to JSON file to load. If file does not exist, and the path begins
        with CONFIG_DIR (`/opt/intelmq/etc` by default), the file from it's
        package data is used.

    Returns:
    --------
    config : dict
        Parsed configuration
    """
    if os.path.exists(configuration_filepath):
        with open(configuration_filepath, 'r') as fpconfig:
            config = json.loads(fpconfig.read())
    elif configuration_filepath.find(intelmq.CONFIG_DIR) == 0:  # at beginning
        newpath = pkg_resources.resource_filename('intelmq', 'etc/')
        filepath = configuration_filepath.replace(intelmq.CONFIG_DIR,
                                                  newpath)
        if os.path.exists(filepath):
            with open(filepath, 'r') as fpconfig:
                config = json.loads(fpconfig.read())
    return config


def load_parameters(*configs):
    """
    Load dictionaries into new Parameters() instance.

    Parameters:
    -----------
    *configs : dict
        Arbitrary number of dictionaries to load.

    Returns:<commit_msg>BUG: utils configuration load: raise if not extist<commit_after># -*- coding: utf-8 -*-
"""
Common utility functions for intelmq.

decode
encode
base64_decode
base64_encode
load_configuration
log
reverse_readline
parse_logline
"""
import base64
import json
import logging
import logging.handlers
import os
import re
import sys
import traceback

import pkg_resources

import intelmq

__all__ = ['base64_decode', 'base64_encode', 'decode', 'encode',
           'load_configuration', 'load_parameters', 'log', 'parse_logline',
           'reverse_readline', 'error_message_from_exc',
           ]

# Used loglines format
LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
LOG_FORMAT_STREAM = '%(name)s: %(message)s'

# Regex for parsing the above LOG_FORMAT
LOG_REGEX = (r'^(?P<date>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d+) -'
             r' (?P<bot_id>[-\w]+) - '
             r'(?P<log_level>[A-Z]+) - '
             r'(?P<message>.+)$')


class Parameters(object):
    pass


def decode(text, encodings=("utf-8", ), force=False):
    """
    Decode given string to UTF-8 (default).

    Parameters:
    -----------
    text : bytes string
        if unicode string is given, same object is returned
    encodings : iterable of strings
        list/tuple of encodings to use, default ('utf-8')
    force : boolean
        Ignore invalid characters, default: False

    Returns
    -------
    text : unicode string
        unicode string is always returned, even when encoding is ascii
        (Python 3 compat)
    """
    if isinstance(text, str):
        return text

    for encoding in encodings:
        try:
            return str(text.decode(encoding))
        except ValueError:
            pass

    if force:
        for encoding in encodings:
            try:
                return str(text.decode(encoding, 'ignore'))
            except ValueError:
                pass

    raise ValueError("Could not decode string with given encodings{!r}"
                     ".".format(encodings))


def encode(text, encodings=("utf-8", ), force=False):
    """
    Encode given string from UTF-8 (default).

    Parameters:
    -----------
    text : unicode string
        if bytes string is given, same object is returned
    encodings : iterable of strings
        list/tuple of encodings to use, default ('utf-8')
    force : boolean
        Ignore invalid characters, default: False
    """
    if isinstance(text, bytes):
        return text

    for encoding in encodings:
        try:
            return text.encode(encoding)
        except ValueError:
            pass

    if force:
        for encoding in encodings:
            try:
                return text.encode(encoding, 'ignore')
            except ValueError:
                pass

    raise ValueError("Could not encode string with given encodings{!r}"
                     ".".format(encodings))


def base64_decode(value):
    """
    Parameters
    ----------
    value : string
        base 64, will be encoded to bytes if not already.

    Returns
    -------
    retval : unicode string

    Notes
    -----
    Possible bytes - unicode conversions problems are ignored.
    """
    return decode(base64.b64decode(encode(value, force=True)), force=True)


def base64_encode(value):
    """
    Parameters
    ----------
    value : string
        Will be encoded to bytes if not already of type bytes.

    Returns
    -------
    retval : unicode string

    Notes
    -----
    Possible bytes - unicode conversions problems are ignored.
    """
    return decode(base64.b64encode(encode(value, force=True)), force=True)


def load_configuration(configuration_filepath):
    """
    Load JSON configuration file.

    Parameters:
    -----------
    configuration_filepath : string
        Path to JSON file to load. If file does not exist, and the path begins
        with CONFIG_DIR (`/opt/intelmq/etc` by default), the file from it's
        package data is used.

    Returns:
    --------
    config : dict
        Parsed configuration
    """
    if os.path.exists(configuration_filepath):
        with open(configuration_filepath, 'r') as fpconfig:
            config = json.loads(fpconfig.read())
    elif configuration_filepath.find(intelmq.CONFIG_DIR) == 0:  # at beginning
        newpath = pkg_resources.resource_filename('intelmq', 'etc/')
        filepath = configuration_filepath.replace(intelmq.CONFIG_DIR,
                                                  newpath)
        if os.path.exists(filepath):
            with open(filepath, 'r') as fpconfig:
                config = json.loads(fpconfig.read())
    else:
        raise ValueError('File not found: %r.' % configuration_filepath)
    return config


def load_parameters(*configs):
    """
    Load dictionaries into new Parameters() instance.

    Parameters:
    -----------
    *configs : dict
        Arbitrary number of dictionaries to load.

    Returns:
{'fraction_non_alphanumeric': 0.11622897004513746, 'fraction_numerical': 0.008001641362330735, 'mean_word_length': 2.1337190613950496, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2840560', 'n_tokens_mistral': 2815, 'n_tokens_neox': 2670, 'n_words': 815}