<commit_before># Copyright 2008-2009 ITA Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Query and friends.

All requests are defined as a Query class which is a Runnable.
"""

import os
import re
import signal

from twisted.internet import reactor, defer, protocol, process
from twisted.internet import error as neterror
from twisted.web import error as weberror
from twisted.web.client import HTTPClientFactory
from twisted.python.util import InsensitiveDict
from twisted.python import failure
from coil import struct

# SSL support is screwy
try:
   from twisted.internet import ssl
except ImportError:
   # happens the first time the interpreter tries to import it
   ssl = None
if ssl and not ssl.supported:
   # happens second and later times
   ssl = None

from nagcat import scheduler, util, log

_queries = {}

def addQuery(conf):
    """Create a new query and register it or return an existing one"""

    qtype = conf.get('type')

    # Find the correct Query class for this type
    qclass = globals().get('Query_%s' % qtype, None)
    if qclass is not None:
        qobj = qclass(conf)
    else:
        raise Exception("Unknown query type '%s'" % qtype)

    key = <KEY>
    if key in _queries:
        log.debug("Reusing query '%s'", key)
        qobj = _queries[key]
    else:
        log.debug("Adding query '%s'", key)
        _queries[key] = qobj

    return qobj


class Query(scheduler.Runnable):
    """Query objects make a single request or run a single process as
    defined in its configuration. The only state they may contain when
    it is not running is the results from the last run (be it real data
    or a Failure object).

    All state that defines a query *MUST* be saved on self.conf and
    never changed after __init__ to allow identical queries to be
    identified reliably.

    Query objects are only used by SimpleTest objects.
    """

    def __init__(self, conf):
        assert isinstance(conf, struct.Struct)
        conf.expand(recursive=False)
        host = conf.get('host', None)
        scheduler.Runnable.__init__(self, conf.get('repeat', None), host)

        # self.conf must contain all configuration variables that
        # this object uses so identical Queries can be identified.
        self.conf = {}

        # All queries should handle timeouts
        try:
            self.conf['timeout'] = float(conf.get('timeout', 15))
        except ValueError:
            raise util.KnownError("Invalid timeout value '%s'" %
                    conf.get('timeout'))

        if self.conf['timeout'] <= 0:
            raise util.KnownError("Non-positive timeout value '%s'" %
                    conf.get('timeout'))

    def _failure_tcp(self, result):
        """Catch common TCP failures and convert them to a KnownError"""

        if isinstance(result.value, neterror.TimeoutError):
            result = failure.Failure(util.KnownError(
                "TCP handshake timeout", state="CRITICAL", error=result.value))

        elif isinstance(result.value, neterror.ConnectionRefusedError):
            result = failure.Failure(util.KnownError(
                "TCP connection refused", state="CRITICAL", error=result.value))

        return result

    def __repr__(self):
        return "%s(%s)" % (self.__class__.__name__, self.conf)


class Query_noop(Query):
    """Dummy query useful for testing."""

    def __init__(self, conf):
        Query.__init__(self, conf)
        self.conf['data'] = conf.get('data', None)

    def _start(self):
        return defer.succeed(self.conf['data'])

class Query_http(Query):
    """Process an HTTP GET or POST"""

    scheme = "http"<commit_msg>Sometimes the epoll reactor raises ConnectError<commit_after># Copyright 2008-2009 ITA Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Query and friends.

All requests are defined as a Query class which is a Runnable.
"""

import os
import re
import signal

from twisted.internet import reactor, defer, protocol, process
from twisted.internet import error as neterror
from twisted.web import error as weberror
from twisted.web.client import HTTPClientFactory
from twisted.python.util import InsensitiveDict
from twisted.python import failure
from coil import struct

# SSL support is screwy
try:
   from twisted.internet import ssl
except ImportError:
   # happens the first time the interpreter tries to import it
   ssl = None
if ssl and not ssl.supported:
   # happens second and later times
   ssl = None

from nagcat import scheduler, util, log

_queries = {}

def addQuery(conf):
    """Create a new query and register it or return an existing one"""

    qtype = conf.get('type')

    # Find the correct Query class for this type
    qclass = globals().get('Query_%s' % qtype, None)
    if qclass is not None:
        qobj = qclass(conf)
    else:
        raise Exception("Unknown query type '%s'" % qtype)

    key = <KEY>
    if key in _queries:
        log.debug("Reusing query '%s'", key)
        qobj = _queries[key]
    else:
        log.debug("Adding query '%s'", key)
        _queries[key] = qobj

    return qobj


class Query(scheduler.Runnable):
    """Query objects make a single request or run a single process as
    defined in its configuration. The only state they may contain when
    it is not running is the results from the last run (be it real data
    or a Failure object).

    All state that defines a query *MUST* be saved on self.conf and
    never changed after __init__ to allow identical queries to be
    identified reliably.

    Query objects are only used by SimpleTest objects.
    """

    def __init__(self, conf):
        assert isinstance(conf, struct.Struct)
        conf.expand(recursive=False)
        host = conf.get('host', None)
        scheduler.Runnable.__init__(self, conf.get('repeat', None), host)

        # self.conf must contain all configuration variables that
        # this object uses so identical Queries can be identified.
        self.conf = {}

        # All queries should handle timeouts
        try:
            self.conf['timeout'] = float(conf.get('timeout', 15))
        except ValueError:
            raise util.KnownError("Invalid timeout value '%s'" %
                    conf.get('timeout'))

        if self.conf['timeout'] <= 0:
            raise util.KnownError("Non-positive timeout value '%s'" %
                    conf.get('timeout'))

    def _failure_tcp(self, result):
        """Catch common TCP failures and convert them to a KnownError"""

        if isinstance(result.value, neterror.TimeoutError):
            result = failure.Failure(util.KnownError(
                "TCP handshake timeout", state="CRITICAL", error=result.value))

        elif isinstance(result.value, neterror.ConnectionRefusedError):
            result = failure.Failure(util.KnownError(
                "TCP connection refused", state="CRITICAL", error=result.value))

        elif isinstance(result.value, neterror.ConnectError):
            # ConnectError sometimes is used to wrap up various other errors.
            result = failure.Failure(util.KnownError(
                "TCP connection error", state="CRITICAL", error=result.value))

        return result

    def __repr__(self):
        return "%s(%s)" % (self.__class__.__name__, self.conf)


class Query_noop(Query):
    """Dummy query useful for testing."""

    def __init__(self, conf):
        Query.__init__(self, conf)
        self.conf['data'] = conf.get('data', None)

    def _start(self):
        return defer.succeed(self.conf['data'])

class Query_http(Query):
    """Process an HTTP GET or POST"""

    scheme = "http"
{'fraction_non_alphanumeric': 0.09095116871094654, 'fraction_numerical': 0.0034714186530895625, 'mean_word_length': 2.944774075764491, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1321271', 'n_tokens_mistral': 2389, 'n_tokens_neox': 2275, 'n_words': 920}