<commit_before>"""
.. module:: network
   :platform: Unix
   :synopsis: A source module for network checks

.. moduleauthor:: <NAME> <<EMAIL>>
"""

import time

from twisted.internet import defer, reactor
from twisted.web.client import Agent
from twisted.web.http_headers import Headers

from zope.interface import implements

from tensor.interfaces import ITensorSource
from tensor.objects import Source

from tensor.utils import BodyReceiver, fork

class HTTP(Source):
    """Performs an HTTP request

    **Configuration arguments:**
    
    :method: HTTP request method to use
    :type method: str.
    :match: A text string to match in the document when it is correct
    :type match: str.
    
    **Metrics:**

    :(service name).latency: Time to complete request
    """

    implements(ITensorSource)

    @defer.inlineCallbacks
    def get(self):
        agent = Agent(reactor)

        method = self.config.get('method', 'GET')
        url = self.config.get('url', 'http://%s/' % self.hostname)
        match = self.config.get('match', None)
        ua = self.config.get('user-agent', 'Tensor HTTP checker')

        t0 = time.time()<commit_msg>hyphens make everyones life hard<commit_after>"""
.. module:: network
   :platform: Unix
   :synopsis: A source module for network checks

.. moduleauthor:: <NAME> <<EMAIL>>
"""

import time

from twisted.internet import defer, reactor
from twisted.web.client import Agent
from twisted.web.http_headers import Headers

from zope.interface import implements

from tensor.interfaces import ITensorSource
from tensor.objects import Source

from tensor.utils import BodyReceiver, fork

class HTTP(Source):
    """Performs an HTTP request

    **Configuration arguments:**
    
    :method: HTTP request method to use
    :type method: str.
    :match: A text string to match in the document when it is correct
    :type match: str.
    :useragent: User-Agent header to use
    :type useragent: str.

    **Metrics:**

    :(service name).latency: Time to complete request
    """

    implements(ITensorSource)

    @defer.inlineCallbacks
    def get(self):
        agent = Agent(reactor)

        method = self.config.get('method', 'GET')
        url = self.config.get('url', 'http://%s/' % self.hostname)
        match = self.config.get('match', None)
        ua = self.config.get('useragent', 'Tensor HTTP checker')

        t0 = time.time()
{'fraction_non_alphanumeric': 0.11213005418924551, 'fraction_numerical': 0.0008336807002917883, 'mean_word_length': 3.210526315789474, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5906789', 'n_tokens_mistral': 695, 'n_tokens_neox': 675, 'n_words': 231}