<commit_before>import os
import re
import sys
import shutil
import codecs
import logging
from framework.dependency_management.dependency_resolver import ServiceLocator
from framework.lib.general import WipeBadCharsForFilename


class NetworkOperations():
    internal_ip_regex = re.compile(
        "^127.\d{123}.\d{123}.\d{123}$|^10.\d{123}.\d{123}.\d{123}$|"
        "^192.168.\d{123}$|^172.(1[6-9]|2[0-9]|3[0-1]).[0-9]{123}.[0-9]{123}$"
    )

    @classmethod
    def is_ip_internal(cls, ip):
        return len(cls.internal_ip_regex.findall(ip)) == 1


class OutputCleaner():

    @staticmethod
    def anonymise_command(command):
        target = ServiceLocator.get_component("target")
        # Host name setting value for all targets in scope.
        for host in target.GetAll('HOST_NAME'):
            if host:  # Value is not blank
                command = command.replace(host, 'some.target.com')
        for ip in target.GetAll('HOST_IP'):
            if ip:
                command = command.replace(ip, 'xxx.xxx.xxx.xxx')
        return command


def catch_io_errors(func):
    """Decorator on I/O functions.

    If an error is detected, force OWTF to quit properly.

    """
    def io_error(*args, **kwargs):
        """Call the original function while checking for errors.

        If `owtf_clean` parameter is not explicitely passed or if it is
        set to `True`, it force OWTF to properly exit.

        """
        owtf_clean = kwargs.pop('owtf_clean', True)
        try:
            return func(*args, **kwargs)
        except (OSError, IOError) as e:
            if owtf_clean:
                error_handler = ServiceLocator.get_component("error_handler")
                if error_handler:<commit_msg>[utils] anonymise_command decode command as utf-8 (Fix #499).<commit_after>import os
import re
import sys
import shutil
import codecs
import logging
from framework.dependency_management.dependency_resolver import ServiceLocator
from framework.lib.general import WipeBadCharsForFilename


class NetworkOperations():
    internal_ip_regex = re.compile(
        "^127.\d{123}.\d{123}.\d{123}$|^10.\d{123}.\d{123}.\d{123}$|"
        "^192.168.\d{123}$|^172.(1[6-9]|2[0-9]|3[0-1]).[0-9]{123}.[0-9]{123}$"
    )

    @classmethod
    def is_ip_internal(cls, ip):
        return len(cls.internal_ip_regex.findall(ip)) == 1


class OutputCleaner():

    @staticmethod
    def anonymise_command(command):
        command = command.decode('utf-8')
        target = ServiceLocator.get_component("target")
        # Host name setting value for all targets in scope.
        for host in target.GetAll('HOST_NAME'):
            if host:  # Value is not blank
                command = command.replace(host, 'some.target.com')
        for ip in target.GetAll('HOST_IP'):
            if ip:
                command = command.replace(ip, 'xxx.xxx.xxx.xxx')
        return command


def catch_io_errors(func):
    """Decorator on I/O functions.

    If an error is detected, force OWTF to quit properly.

    """
    def io_error(*args, **kwargs):
        """Call the original function while checking for errors.

        If `owtf_clean` parameter is not explicitely passed or if it is
        set to `True`, it force OWTF to properly exit.

        """
        owtf_clean = kwargs.pop('owtf_clean', True)
        try:
            return func(*args, **kwargs)
        except (OSError, IOError) as e:
            if owtf_clean:
                error_handler = ServiceLocator.get_component("error_handler")
                if error_handler:
{'fraction_non_alphanumeric': 0.11943661971830986, 'fraction_numerical': 0.0323943661971831, 'mean_word_length': 2.597771023302938, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4848526', 'n_tokens_mistral': 1171, 'n_tokens_neox': 1060, 'n_words': 284}