<commit_before>"""
    Server Density
    www.serverdensity.com
    ----
    Server monitoring agent for Linux, FreeBSD and Mac OS X

    Licensed under Simplified BSD License (see LICENSE)
"""

import httplib  # Used only for handling httplib.HTTPException
import datetime
import json
import os
import platform
import socket
import string
import threading
import time
import urllib
import urllib2

pythonVersion = platform.python_version_tuple()
python24 = platform.python_version().startswith('2.4')

# Build the request headers
headers = {
    'User-Agent': 'Server Density Agent',
    'Content-Type': 'application/x-www-form-urlencoded',
    'Accept': 'text/html, */*',
}

if int(pythonVersion[1]) >= 6:  # Don't bother checking major version since we only support v2 anyway
    import json
else:
    import minjson


class LogTailer(threading.Thread):

    def __init__(self, agentConfig, mainLogger, filename):
        self.agentConfig = agentConfig
        self.mainLogger = mainLogger
        self.filename = filename

        threading.Thread.__init__(self)

    # http://www.dabeaz.com/generators/follow.py
    def follow(self, thefile):
        try:
            thefile.seek(0, 2)

            while True:

                line = thefile.readline()
                if not line:
                    time.sleep(0.1)

                    # Check to see if the file is still the one we opened
                    # This can change if the log gets rotated, for example
                    fstat = os.fstat(thefile.fileno())
                    stat = os.stat(self.filename)

                    if fstat.st_ino != stat.st_ino:
                        self.mainLogger.warn('LogTailer (%s) - log file changed, reopening', self.filename)

                        thefile.close()

                        thefile = open(self.filename, "r")

                    continue

                yield line

        except Exception:
            import traceback
            self.mainLogger.error('LogTailer (%s) - follow() failed = %s', self.filename, traceback.format_exc())
            return

    def run(self):

        try:
            filename = open(self.filename, "r")
            loglines = self.follow(filename)

        except Exception:
            import traceback
            self.mainLogger.error('LogTailer (%s) - Unable to open log file = %s', self.filename, traceback.format_exc())
            return

        for line in loglines:
            payload = {
                "itemId": self.agentConfig['agentKey'],
                "logs":
                [{
                    "timestamp": datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S"),
                    "message": line,
                    "filename": self.filename
                }]
            }

            try:
                if int(pythonVersion[1]) >= 6:
                    payloadJSON = json.dumps(payload)
                else:
                    payloadJSON = minjson.write(checksData)

            except Exception:
                import traceback
                self.mainLogger.error('LogTailer (%s) - Failed encoding payload to json. Exception = %s', self.filename, traceback.format_exc())
                return

            sdUrl = string.replace(self.agentConfig['sdUrl'], 'https://', '')
            sdUrl = string.replace(sdUrl, '.serverdensity.io', '')

            payload = {'payload': payloadJSON, 'sdUrl': sdUrl}
            self.doPostBack(payload)

    def doPostBack(self, postBackData, retry=False):

        self.mainLogger.debug('LogTailer (%s) - doPostBack: start', self.filename)

        try:

            try:
                self.mainLogger.debug('LogTailer (%s) - doPostBack: attempting postback for %s', self.filename, postBackData['sdUrl'])

                # Build the request handler
                request = urllib2.Request('https://logs.serverdensity.io/collector', urllib.urlencode(postBackData), headers)

                # Do the request, log any errors
                response = urllib2.urlopen(request)

                self.mainLogger.info('LogTailer (%s) - Postback response: %s', self.filename, response.read())

            except urllib2.HTTPError, e:
                self.mainLogger.error('LogTailer (%s) - doPostBack: HTTPError = %s', self.filename, e)

                return

            except urllib2.URLError, e:
                self.mainLogger.error('LogTailer (%s) - doPostBack: URLError = %s', self.filename, e)

                # attempt a lookup, in case of DNS fail
                # https://github.com/serverdensity/sd-agent/issues/47
                if not retry:

                    timeout = socket.getdefaulttimeout()
                    socket.setdefaulttimeout(5)

                    self.mainLogger.info('LogTailer (%s) - doPostBack: Retrying postback with DNS lookup iteration', self.filename)
                    try:
                        [socket.gethostbyname(self.agentConfig['sdUrl']) for x in xrange(0, 2)]
                    except Exception:
                        # this can raise, if the dns lookup doesn't work
                        pass
                    socket.setdefaulttimeout(timeout)

                    self.mainLogger.info("LogTailer (%s) - doPostBack: Executing retry", self.filename)
                    return self.doPostBack(postBackData, retry=True)
                else:
                    # if we get here, the retry has failed, so we need to reschedule
                    self.mainLogger.info("LogTailer (%s) - doPostBack: Retry failed, rescheduling", self.filename)

                    return

            except httplib.HTTPException, e:  # Added for case #26701
                self.mainLogger.error('LogTailer (%s) - doPostBack: HTTPException = %s', self.filename, e)
                return

            except Exception:
                import traceback
                self.mainLogger.error('LogTailer (%s) - doPostBack: Exception = %s', self.filename, traceback.format_exc())
                return

        finally:
            if int(pythonVersion[1]) >= 6:
                try:
                    if 'response' in locals():
                        response.close()
                except Exception:
                    import traceback
                    self.mainLogger.error('LogTailer (%s) - doPostBack: Exception = %s', self.filename, traceback.format_exc())
                    return

            self.mainLogger.debug('LogTailer (%s) - doPostBack: completed', self.filename)<commit_msg>Shorten log lines<commit_after>"""
    Server Density
    www.serverdensity.com
    ----
    Server monitoring agent for Linux, FreeBSD and Mac OS X

    Licensed under Simplified BSD License (see LICENSE)
"""

import httplib  # Used only for handling httplib.HTTPException
import datetime
import json
import os
import platform
import socket
import string
import threading
import time
import urllib
import urllib2

pythonVersion = platform.python_version_tuple()
python24 = platform.python_version().startswith('2.4')

# Build the request headers
headers = {
    'User-Agent': 'Server Density Agent',
    'Content-Type': 'application/x-www-form-urlencoded',
    'Accept': 'text/html, */*',
}

if int(pythonVersion[1]) >= 6:  # Don't bother checking major version since we only support v2 anyway
    import json
else:
    import minjson


class LogTailer(threading.Thread):

    def __init__(self, agentConfig, mainLogger, filename):
        self.agentConfig = agentConfig
        self.mainLogger = mainLogger
        self.filename = filename

        threading.Thread.__init__(self)

    # http://www.dabeaz.com/generators/follow.py
    def follow(self, thefile):
        try:
            thefile.seek(0, 2)

            while True:

                line = thefile.readline()
                if not line:
                    time.sleep(0.1)

                    # Check to see if the file is still the one we opened
                    # This can change if the log gets rotated, for example
                    fstat = os.fstat(thefile.fileno())
                    stat = os.stat(self.filename)

                    if fstat.st_ino != stat.st_ino:
                        self.mainLogger.warn('LogTailer (%s) - log file changed, reopening', self.filename)

                        thefile.close()

                        thefile = open(self.filename, "r")

                    continue

                yield line

        except Exception:
            import traceback
            self.mainLogger.error(
                'LogTailer (%s) - follow() failed = %s',
                self.filename,
                traceback.format_exc()
            )
            return

    def run(self):

        try:
            filename = open(self.filename, "r")
            loglines = self.follow(filename)

        except Exception:
            import traceback
            self.mainLogger.error(
                'LogTailer (%s) - Unable to open log file = %s',
                self.filename,
                traceback.format_exc()
            )
            return

        for line in loglines:
            payload = {
                "itemId": self.agentConfig['agentKey'],
                "logs":
                [{
                    "timestamp": datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S"),
                    "message": line,
                    "filename": self.filename
                }]
            }

            try:
                if int(pythonVersion[1]) >= 6:
                    payloadJSON = json.dumps(payload)
                else:
                    payloadJSON = minjson.write(checksData)

            except Exception:
                import traceback
                self.mainLogger.error(
                    'LogTailer (%s) - Failed encoding payload to json. Exception = %s',
                    self.filename,
                    traceback.format_exc()
                )
                return

            sdUrl = string.replace(self.agentConfig['sdUrl'], 'https://', '')
            sdUrl = string.replace(sdUrl, '.serverdensity.io', '')

            payload = {'payload': payloadJSON, 'sdUrl': sdUrl}
            self.doPostBack(payload)

    def doPostBack(self, postBackData, retry=False):

        self.mainLogger.debug('LogTailer (%s) - doPostBack: start', self.filename)

        try:

            try:
                self.mainLogger.debug(
                    'LogTailer (%s) - doPostBack: attempting postback for %s',
                    self.filename,
                    postBackData['sdUrl']
                )

                # Build the request handler
                request = urllib2.Request(
                    'https://logs.serverdensity.io/collector',
                    urllib.urlencode(postBackData),
                    headers
                )

                # Do the request, log any errors
                response = urllib2.urlopen(request)

                self.mainLogger.info(
                    'LogTailer (%s) - Postback response: %s',
                    self.filename,
                    response.read()
                )

            except urllib2.HTTPError, e:
                self.mainLogger.error('LogTailer (%s) - doPostBack: HTTPError = %s', self.filename, e)

                return

            except urllib2.URLError, e:
                self.mainLogger.error('LogTailer (%s) - doPostBack: URLError = %s', self.filename, e)

                # attempt a lookup, in case of DNS fail
                # https://github.com/serverdensity/sd-agent/issues/47
                if not retry:

                    timeout = socket.getdefaulttimeout()
                    socket.setdefaulttimeout(5)

                    self.mainLogger.info(
                        'LogTailer (%s) - doPostBack: Retrying postback with DNS lookup iteration',
                        self.filename
                    )
                    try:
                        [socket.gethostbyname(self.agentConfig['sdUrl']) for x in xrange(0, 2)]
                    except Exception:
                        # this can raise, if the dns lookup doesn't work
                        pass
                    socket.setdefaulttimeout(timeout)

                    self.mainLogger.info(
                        "LogTailer (%s) - doPostBack: Executing retry",
                        self.filename
                    )
                    return self.doPostBack(postBackData, retry=True)
                else:
                    # if we get here, the retry has failed, so we need to reschedule
                    self.mainLogger.info(
                        "LogTailer (%s) - doPostBack: Retry failed, rescheduling",
                        self.filename
                    )

                    return

            except httplib.HTTPException, e:  # Added for case #26701
                self.mainLogger.error(
                    'LogTailer (%s) - doPostBack: HTTPException = %s',
                    self.filename,
                    e
                )
                return

            except Exception:
                import traceback
                self.mainLogger.error(
                    'LogTailer (%s) - doPostBack: Exception = %s',
                    self.filename,
                    traceback.format_exc()
                )
                return

        finally:
            if int(pythonVersion[1]) >= 6:
                try:
                    if 'response' in locals():
                        response.close()
                except Exception:
                    import traceback
                    self.mainLogger.error(
                        'LogTailer (%s) - doPostBack: Exception = %s',
                        self.filename,
                        traceback.format_exc()
                    )
                    return

            self.mainLogger.debug('LogTailer (%s) - doPostBack: completed', self.filename)
{'fraction_non_alphanumeric': 0.0942174913693901, 'fraction_numerical': 0.00431530494821634, 'mean_word_length': 1.445910290237467, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 6, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '441646', 'n_tokens_mistral': 3738, 'n_tokens_neox': 3445, 'n_words': 885}