<commit_before># stdlib
import atexit
import cStringIO as StringIO
from collections import namedtuple
from functools import partial
import glob
try:
    import grp
except ImportError:
    # The module only exists on Unix platforms
    grp = None
import logging
import os
try:
    import pwd
except ImportError:
    # Same as above (exists on Unix platforms only)
    pwd = None
import re
import stat
import subprocess
import sys
import tarfile
import tempfile
from time import strftime
import traceback

# 3p
import requests

# DD imports
from checks.check_status import CollectorStatus, DogstatsdStatus, ForwarderStatus
from config import (
    check_yaml,
    get_confd_path,
    get_config,
    get_config_path,
    get_logging_config,
    get_url_endpoint,
)
from jmxfetch import JMXFetch
from util import get_hostname
from utils.jmx import jmx_command, JMXFiles
from utils.platform import Platform

# Globals
log = logging.getLogger(__name__)


def configcheck():
    all_valid = True
    for conf_path in glob.glob(os.path.join(get_confd_path(), "*.yaml")):
        basename = os.path.basename(conf_path)
        try:
            check_yaml(conf_path)
        except Exception, e:
            all_valid = False
            print "%s contains errors:\n    %s" % (basename, e)
        else:
            print "%s is valid" % basename
    if all_valid:
        print "All yaml files passed. You can now run the Datadog agent."
        return 0
    else:
        print("Fix the invalid yaml files above in order to start the Datadog agent. "
              "A useful external tool for yaml parsing can be found at "
              "http://yaml-online-parser.appspot.com/")
        return 1


class Flare(object):
    """
    Compress all important logs and configuration files for debug,
    and then send them to Datadog (which transfers them to Support)
    """

    DATADOG_SUPPORT_URL = '/support/flare'

    CredentialPattern = namedtuple('CredentialPattern', ['pattern', 'replacement', 'label'])
    CHECK_CREDENTIALS = [
        CredentialPattern(
            re.compile('( *(\w|_)*pass(word)?:).+'),
            r'\1 ********',
            'password'
        ),
        CredentialPattern(
            re.compile('(.*\ [A-Za-z0-9]+)\:\/\/([A-Za-z0-9]+)\:(.+)\@'),
            r'\1://\2:********@',
            'password in <PASSWORD>'
        ),
    ]
    MAIN_CREDENTIALS = [
        CredentialPattern(
            re.compile('^api_key: *\w+(\w{5})$'),
            r'api_key: *************************\1',
            'api_key'
        ),
    ]
    COMMENT_REGEX = re.compile('^ *#.*')

    COMPRESSED_FILE = 'datadog-agent-{0}.tar.bz2'
    # We limit to 10MB arbitrarily
    MAX_UPLOAD_SIZE = 10485000
    TIMEOUT = 60

    def __init__(self, cmdline=False, case_id=None):
        self._case_id = case_id
        self._cmdline = cmdline
        self._init_tarfile()
        self._init_permissions_file()
        self._save_logs_path()
        self._config = get_config()
        self._api_key = self._config.get('api_key')
        self._url = "{0}{1}".format(<commit_msg>[flare] Remove proxy credentials from collected `datadog.conf`<commit_after># stdlib
import atexit
import cStringIO as StringIO
from collections import namedtuple
from functools import partial
import glob
try:
    import grp
except ImportError:
    # The module only exists on Unix platforms
    grp = None
import logging
import os
try:
    import pwd
except ImportError:
    # Same as above (exists on Unix platforms only)
    pwd = None
import re
import stat
import subprocess
import sys
import tarfile
import tempfile
from time import strftime
import traceback

# 3p
import requests

# DD imports
from checks.check_status import CollectorStatus, DogstatsdStatus, ForwarderStatus
from config import (
    check_yaml,
    get_confd_path,
    get_config,
    get_config_path,
    get_logging_config,
    get_url_endpoint,
)
from jmxfetch import JMXFetch
from util import get_hostname
from utils.jmx import jmx_command, JMXFiles
from utils.platform import Platform

# Globals
log = logging.getLogger(__name__)


def configcheck():
    all_valid = True
    for conf_path in glob.glob(os.path.join(get_confd_path(), "*.yaml")):
        basename = os.path.basename(conf_path)
        try:
            check_yaml(conf_path)
        except Exception, e:
            all_valid = False
            print "%s contains errors:\n    %s" % (basename, e)
        else:
            print "%s is valid" % basename
    if all_valid:
        print "All yaml files passed. You can now run the Datadog agent."
        return 0
    else:
        print("Fix the invalid yaml files above in order to start the Datadog agent. "
              "A useful external tool for yaml parsing can be found at "
              "http://yaml-online-parser.appspot.com/")
        return 1


class Flare(object):
    """
    Compress all important logs and configuration files for debug,
    and then send them to Datadog (which transfers them to Support)
    """

    DATADOG_SUPPORT_URL = '/support/flare'

    CredentialPattern = namedtuple('CredentialPattern', ['pattern', 'replacement', 'label'])
    CHECK_CREDENTIALS = [
        CredentialPattern(
            re.compile('( *(\w|_)*pass(word)?:).+'),
            r'\1 ********',
            'password'
        ),
        CredentialPattern(
            re.compile('(.*\ [A-Za-z0-9]+)\:\/\/([A-Za-z0-9]+)\:(.+)\@'),
            r'\1://\2:********@',
            'password in a uri'
        ),
    ]
    MAIN_CREDENTIALS = [
        CredentialPattern(
            re.compile('^api_key: *\w+(\w{5})$'),
            r'api_key: *************************\1',
            'api_key'
        ),
        CredentialPattern(
            re.compile('^(proxy_user|proxy_password): *.+'),
            r'\1: ********',
            'proxy credentials'
        ),
    ]
    COMMENT_REGEX = re.compile('^ *#.*')

    COMPRESSED_FILE = 'datadog-agent-{0}.tar.bz2'
    # We limit to 10MB arbitrarily
    MAX_UPLOAD_SIZE = 10485000
    TIMEOUT = 60

    def __init__(self, cmdline=False, case_id=None):
        self._case_id = case_id
        self._cmdline = cmdline
        self._init_tarfile()
        self._init_permissions_file()
        self._save_logs_path()
        self._config = get_config()
        self._api_key = self._config.get('api_key')
        self._url = "{0}{1}".format(
{'fraction_non_alphanumeric': 0.11863607793840351, 'fraction_numerical': 0.008956631049654306, 'mean_word_length': 2.4934138309549945, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5781857', 'n_tokens_mistral': 2022, 'n_tokens_neox': 1933, 'n_words': 537}