<commit_before>import ConfigParser
import os
import itertools
import logging
import logging.config
import logging.handlers
import platform
import string
import subprocess
import sys
import glob
import inspect
import traceback
import re
import imp
import socket
from socket import gaierror
from optparse import OptionParser, Values
from cStringIO import StringIO

# project
from util import get_os, yaml, yLoader, Platform
from jmxfetch import JMXFetch, JMX_COLLECT_COMMAND
from migration import migrate_old_style_configuration

# CONSTANTS
DATADOG_CONF = "datadog.conf"
DEFAULT_CHECK_FREQUENCY = 15   # seconds
DEFAULT_STATSD_FREQUENCY = 2  # seconds
DEFAULT_STATSD_BUCKET_SIZE = 10 #seconds
PUP_STATSD_FREQUENCY = 2       # seconds
PUP_STATSD_BUCKET_SIZE = 2       # seconds
LOGGING_MAX_BYTES = 5 * 1024 * 1024

log = logging.getLogger(__name__)
windows_file_handler_added = False

class PathNotFound(Exception):
    pass


def get_parsed_args():
    parser = OptionParser()
    parser.add_option('-d', '--dd_url', action='store', default=None,
                        dest='dd_url')
    parser.add_option('-c', '--clean', action='store_true', default=False,
                        dest='clean')
    parser.add_option('-u', '--use-local-forwarder', action='store_true',
                        default=False, dest='use_forwarder')
    parser.add_option('-n', '--disable-dd', action='store_true', default=False,
                        dest="disable_dd")
    parser.add_option('-v', '--verbose', action='store_true', default=False,
                        dest='verbose',
                      help='Print out stacktraces for errors in checks')

    try:
        options, args = parser.parse_args()
    except SystemExit:
        # Ignore parse errors
        options, args = Values({'dd_url': None,
                                'clean': False,
                                'disable_dd':False,
                                'use_forwarder': False}), []
    return options, args


def get_version():
    return "4.2.2"

def skip_leading_wsp(f):
    "Works on a file, returns a file-like object"
    return StringIO("\n".join(map(string.strip, f.readlines())))


def _windows_commondata_path():
    """Return the common appdata path, using ctypes
    From http://stackoverflow.com/questions/626796/\
    how-do-i-find-the-windows-common-application-data-folder-using-python
    """
    import ctypes
    from ctypes import wintypes, windll

    CSIDL_COMMON_APPDATA = 35

    _SHGetFolderPath = windll.shell32.SHGetFolderPathW
    _SHGetFolderPath.argtypes = [wintypes.HWND,
                                ctypes.c_int,
                                wintypes.HANDLE,
                                wintypes.DWORD, wintypes.LPCWSTR]

    path_buf = wintypes.create_unicode_buffer(wintypes.MAX_PATH)
    result = _SHGetFolderPath(0, CSIDL_COMMON_APPDATA, 0, 0, path_buf)
    return path_buf.value


def _windows_config_path():
    common_data = _windows_commondata_path()
    path = os.path.join(common_data, 'Datadog', DATADOG_CONF)
    if os.path.exists(path):
        return path
    raise PathNotFound(path)


def _windows_confd_path():
    common_data = _windows_commondata_path()
    path = os.path.join(common_data, 'Datadog', 'conf.d')
    if os.path.exists(path):
        return path
    raise PathNotFound(path)


def _windows_checksd_path():
    if hasattr(sys, 'frozen'):
        # we're frozen - from py2exe
        prog_path = os.path.dirname(sys.executable)
        checksd_path = os.path.join(prog_path, '..', 'checks.d')
    else:

        cur_path = os.path.dirname(__file__)
        checksd_path = os.path.join(cur_path, 'checks.d')

    if os.path.exists(checksd_path):
        return checksd_path
    raise PathNotFound(checksd_path)


def _unix_config_path():
    path = os.path.join('/etc/dd-agent', DATADOG_CONF)
    if os.path.exists(path):
        return path
    raise PathNotFound(path)


def _unix_confd_path():
    path = os.path.join('/etc/dd-agent', 'conf.d')
    if os.path.exists(path):
        return path
    raise PathNotFound(path)


def _unix_checksd_path():
    # Unix only will look up based on the current directory
    # because checks.d will hang with the other python modules
    cur_path = os.path.dirname(os.path.realpath(__file__))
    checksd_path = os.path.join(cur_path, 'checks.d')

    if os.path.exists(checksd_path):
        return checksd_path
    raise PathNotFound(checksd_path)


def _is_affirmative(s):
    return s.lower() in ('yes', 'true', '1')


def get_config_path(cfg_path=None, os_name=None):
    # Check if there's an override and if it exists
    if cfg_path is not None and os.path.exists(cfg_path):
        return cfg_path

    if os_name is None:
        os_name = get_os()

    # Check for an OS-specific path, continue on not-found exceptions
    bad_path = ''
    if os_name == 'windows':
        try:
            return _windows_config_path()
        except PathNotFound, e:
            if len(e.args) > 0:
                bad_path = e.args[0]
    else:
        try:
            return _unix_config_path()
        except PathNotFound, e:
            if len(e.args) > 0:
                bad_path = e.args[0]

    # Check if there's a config stored in the current agent directory
    path = os.path.realpath(__file__)
    path = os.path.dirname(path)
    if os.path.exists(os.path.join(path, DATADOG_CONF)):
        return os.path.join(path, DATADOG_CONF)

    # If all searches fail, exit the agent with an error
    sys.stderr.write("Please supply a configuration file at %s or in the directory where the Agent is currently deployed.\n" % bad_path)
    sys.exit(3)

def get_default_bind_host():
    try:
        socket.gethostbyname('localhost')
    except gaierror:
        log.warning("localhost seems undefined in your hosts file, using 127.0.0.1 instead")
        return '127.0.0.1'
    return 'locahost'

def get_config(parse_args=True, cfg_path=None, options=None):
    if parse_args:
        options, _ = get_parsed_args()

    # General config
    agentConfig = {
        'check_freq': DEFAULT_CHECK_FREQUENCY,
        'dogstatsd_interval': DEFAULT_STATSD_FREQUENCY,
        'dogstatsd_agregator_bucket_size': DEFAULT_STATSD_BUCKET_SIZE,
        'dogstatsd_normalize': 'yes',
        'dogstatsd_port': 8125,
        'dogstatsd_target': 'http://localhost:17123',
        'graphite_listen_port': None,
        'hostname': None,
        'listen_port': None,
        'tags': None,
        'use_ec2_instance_id': False,  # DEPRECATED
        'version': get_version(),
        'watchdog': True,
        'additional_checksd': '/etc/dd-agent/checks.d/',
        'bind_host': get_default_bind_host(),
    }

    dogstatsd_interval = DEFAULT_STATSD_FREQUENCY
    dogstatsd_agregator_bucket_size = DEFAULT_STATSD_BUCKET_SIZE

    # Config handling<commit_msg>Config| Fix typo for default_bind_host<commit_after>import ConfigParser
import os
import itertools
import logging
import logging.config
import logging.handlers
import platform
import string
import subprocess
import sys
import glob
import inspect
import traceback
import re
import imp
import socket
from socket import gaierror
from optparse import OptionParser, Values
from cStringIO import StringIO

# project
from util import get_os, yaml, yLoader, Platform
from jmxfetch import JMXFetch, JMX_COLLECT_COMMAND
from migration import migrate_old_style_configuration

# CONSTANTS
DATADOG_CONF = "datadog.conf"
DEFAULT_CHECK_FREQUENCY = 15   # seconds
DEFAULT_STATSD_FREQUENCY = 2  # seconds
DEFAULT_STATSD_BUCKET_SIZE = 10 #seconds
PUP_STATSD_FREQUENCY = 2       # seconds
PUP_STATSD_BUCKET_SIZE = 2       # seconds
LOGGING_MAX_BYTES = 5 * 1024 * 1024

log = logging.getLogger(__name__)
windows_file_handler_added = False

class PathNotFound(Exception):
    pass


def get_parsed_args():
    parser = OptionParser()
    parser.add_option('-d', '--dd_url', action='store', default=None,
                        dest='dd_url')
    parser.add_option('-c', '--clean', action='store_true', default=False,
                        dest='clean')
    parser.add_option('-u', '--use-local-forwarder', action='store_true',
                        default=False, dest='use_forwarder')
    parser.add_option('-n', '--disable-dd', action='store_true', default=False,
                        dest="disable_dd")
    parser.add_option('-v', '--verbose', action='store_true', default=False,
                        dest='verbose',
                      help='Print out stacktraces for errors in checks')

    try:
        options, args = parser.parse_args()
    except SystemExit:
        # Ignore parse errors
        options, args = Values({'dd_url': None,
                                'clean': False,
                                'disable_dd':False,
                                'use_forwarder': False}), []
    return options, args


def get_version():
    return "4.2.2"

def skip_leading_wsp(f):
    "Works on a file, returns a file-like object"
    return StringIO("\n".join(map(string.strip, f.readlines())))


def _windows_commondata_path():
    """Return the common appdata path, using ctypes
    From http://stackoverflow.com/questions/626796/\
    how-do-i-find-the-windows-common-application-data-folder-using-python
    """
    import ctypes
    from ctypes import wintypes, windll

    CSIDL_COMMON_APPDATA = 35

    _SHGetFolderPath = windll.shell32.SHGetFolderPathW
    _SHGetFolderPath.argtypes = [wintypes.HWND,
                                ctypes.c_int,
                                wintypes.HANDLE,
                                wintypes.DWORD, wintypes.LPCWSTR]

    path_buf = wintypes.create_unicode_buffer(wintypes.MAX_PATH)
    result = _SHGetFolderPath(0, CSIDL_COMMON_APPDATA, 0, 0, path_buf)
    return path_buf.value


def _windows_config_path():
    common_data = _windows_commondata_path()
    path = os.path.join(common_data, 'Datadog', DATADOG_CONF)
    if os.path.exists(path):
        return path
    raise PathNotFound(path)


def _windows_confd_path():
    common_data = _windows_commondata_path()
    path = os.path.join(common_data, 'Datadog', 'conf.d')
    if os.path.exists(path):
        return path
    raise PathNotFound(path)


def _windows_checksd_path():
    if hasattr(sys, 'frozen'):
        # we're frozen - from py2exe
        prog_path = os.path.dirname(sys.executable)
        checksd_path = os.path.join(prog_path, '..', 'checks.d')
    else:

        cur_path = os.path.dirname(__file__)
        checksd_path = os.path.join(cur_path, 'checks.d')

    if os.path.exists(checksd_path):
        return checksd_path
    raise PathNotFound(checksd_path)


def _unix_config_path():
    path = os.path.join('/etc/dd-agent', DATADOG_CONF)
    if os.path.exists(path):
        return path
    raise PathNotFound(path)


def _unix_confd_path():
    path = os.path.join('/etc/dd-agent', 'conf.d')
    if os.path.exists(path):
        return path
    raise PathNotFound(path)


def _unix_checksd_path():
    # Unix only will look up based on the current directory
    # because checks.d will hang with the other python modules
    cur_path = os.path.dirname(os.path.realpath(__file__))
    checksd_path = os.path.join(cur_path, 'checks.d')

    if os.path.exists(checksd_path):
        return checksd_path
    raise PathNotFound(checksd_path)


def _is_affirmative(s):
    return s.lower() in ('yes', 'true', '1')


def get_config_path(cfg_path=None, os_name=None):
    # Check if there's an override and if it exists
    if cfg_path is not None and os.path.exists(cfg_path):
        return cfg_path

    if os_name is None:
        os_name = get_os()

    # Check for an OS-specific path, continue on not-found exceptions
    bad_path = ''
    if os_name == 'windows':
        try:
            return _windows_config_path()
        except PathNotFound, e:
            if len(e.args) > 0:
                bad_path = e.args[0]
    else:
        try:
            return _unix_config_path()
        except PathNotFound, e:
            if len(e.args) > 0:
                bad_path = e.args[0]

    # Check if there's a config stored in the current agent directory
    path = os.path.realpath(__file__)
    path = os.path.dirname(path)
    if os.path.exists(os.path.join(path, DATADOG_CONF)):
        return os.path.join(path, DATADOG_CONF)

    # If all searches fail, exit the agent with an error
    sys.stderr.write("Please supply a configuration file at %s or in the directory where the Agent is currently deployed.\n" % bad_path)
    sys.exit(3)

def get_default_bind_host():
    try:
        socket.gethostbyname('localhost')
    except gaierror:
        log.warning("localhost seems undefined in your hosts file, using 127.0.0.1 instead")
        return '127.0.0.1'
    return 'localhost'

def get_config(parse_args=True, cfg_path=None, options=None):
    if parse_args:
        options, _ = get_parsed_args()

    # General config
    agentConfig = {
        'check_freq': DEFAULT_CHECK_FREQUENCY,
        'dogstatsd_interval': DEFAULT_STATSD_FREQUENCY,
        'dogstatsd_agregator_bucket_size': DEFAULT_STATSD_BUCKET_SIZE,
        'dogstatsd_normalize': 'yes',
        'dogstatsd_port': 8125,
        'dogstatsd_target': 'http://localhost:17123',
        'graphite_listen_port': None,
        'hostname': None,
        'listen_port': None,
        'tags': None,
        'use_ec2_instance_id': False,  # DEPRECATED
        'version': get_version(),
        'watchdog': True,
        'additional_checksd': '/etc/dd-agent/checks.d/',
        'bind_host': get_default_bind_host(),
    }

    dogstatsd_interval = DEFAULT_STATSD_FREQUENCY
    dogstatsd_agregator_bucket_size = DEFAULT_STATSD_BUCKET_SIZE

    # Config handling
{'fraction_non_alphanumeric': 0.10176539247154946, 'fraction_numerical': 0.008899912459877445, 'mean_word_length': 2.7755439272927567, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3970203', 'n_tokens_mistral': 4503, 'n_tokens_neox': 4320, 'n_words': 1005}