<commit_before># No shebang line, this module is meant to be imported
#
# Copyright 2013 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Manager Service
===============

Sends and receives information from the master and performs systems level tasks
such as log reading, system information gathering, and management of processes.
"""

import logging
import socket
from functools import partial

from zope.interface import implementer
from twisted.python import log, usage
from twisted.plugin import IPlugin
from twisted.application.service import IServiceMaker, MultiService

try:
    import json
except ImportError:
    import simplejson as json

from pyfarm.core.utility import convert
from pyfarm.agent.http.client import post as http_post
from pyfarm.agent.utility.retry import RetryDeferred
from pyfarm.agent.manager.tasks import memory_utilization
from pyfarm.agent.utility.tasks import ScheduledTaskManager


def convert_option_ston(key, value):
    # special "infinite" value reserved for some flags
    if (isinstance(value, basestring) and key == "http-max-retries"
            and value.lower() in ("inf", "infinite", "unlimited")):
        return float("inf")

    try:
        return convert.ston(value)

    except ValueError:
        raise usage.UsageError(
            "--%s requires a number but got %s" % (key, repr(value)))


def check_address(value):
    # is this a valid ip address?
    try:
        socket.inet_aton(value)
        return value

    except socket.error:
        pass

    # could we map the hostname to an address?
    try:
        socket.gethostbyname(value)
        return value

    except socket.gaierror:
        raise usage.UsageError("failed to resolve %s to an address" % value)


class Options(usage.Options):
    optParameters = [
        ("master", "", None,
         "The master server's hostname or address.  If no other options are "
         "provided to describe the location of specific services then the "
         "resource urls will be built off of this address."),
        ("port", "p", 50000,
         "The port which the master should use to talk back to the agent."),
        ("memory-check-interval", "", 10,
         "how often swap and ram resources should be checked and sent to the "
         "master"),
        ("http-max-retries", "", "unlimited",
         "the max number of times to retry a request back to the master"),
        ("http-retry-delay", "", 3,
         "if a http request back to the master has failed, wait this amount of "
         "time before trying again"),
        ("http-auth-user", "", None,
         "the user to use for connecting to the master's REST api.  The "
         "default is communication without authentication."),
        ("http-auth-password", "", None,
         "the password to use to connect to the master's REST api.  The "
         "default is communication without authentication."),
        ("http-api-port", "", 5000,
         "Default port the restfull HTTP api runs on.  By default this value "
         "is combined with --master but could also be combined with the "
         "alternate --http-api-server"),
        ("http-api-scheme", "", "http",
         "The scheme to use to communicate over http.  Valid values are "
         "'http' or 'https'"),
        ("statsd-port", "", 8125,
         "Default port that statsd runs on.  By default this value is combined "
         "with --master but could also be combined with the alternate "
         "--statsd-server"),
        ("http-api-server", "", None,
         "Alternate server which is running the restfull HTTP server.  This "
         "will replace the value provided by the --master flag."),
        ("statsd-server", "", None,
         "Alternate server which is running the statsd service.  This will "
         "replace the value provided by the --master flag.")]

    # special variable used for inferring type in makeService
    optConverters = {
        "port": convert_option_ston,
        "memory-check-interval": convert_option_ston,
        "http-max-retries": convert_option_ston,
        "http-retry-delay": convert_option_ston,
        "http-api-port": convert_option_ston,
        "statsd-port": convert_option_ston}
#
#class IPCReceieverFactory(Factory):
#    """
#    Receives incoming connections and hands them off to the protocol
#    object.  In addition this class will also keep a list of all hosts
#    which have connected so they can be notified upon shutdown.
#    """
#    protocol = IPCProtocol
#    known_hosts = set()
#
#    def stopFactory(self):  # TODO: notify all connected hosts we are stopping
#        if self.known_hosts:<commit_msg>minor structure update, adding projects flag<commit_after># No shebang line, this module is meant to be imported
#
# Copyright 2013 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Manager Service
===============

Sends and receives information from the master and performs systems level tasks
such as log reading, system information gathering, and management of processes.
"""

import logging
import socket
from functools import partial

from zope.interface import implementer
from twisted.python import log, usage
from twisted.plugin import IPlugin
from twisted.application.service import IServiceMaker, MultiService

try:
    import json
except ImportError:
    import simplejson as json

from pyfarm.core.utility import convert
from pyfarm.agent.http.client import post as http_post
from pyfarm.agent.utility.retry import RetryDeferred
from pyfarm.agent.manager.tasks import memory_utilization
from pyfarm.agent.utility.tasks import ScheduledTaskManager


def check_address(value):
    # is this a valid ip address?
    try:
        socket.inet_aton(value)
        return value

    except socket.error:
        pass

    # could we map the hostname to an address?
    try:
        socket.gethostbyname(value)
        return value

    except socket.gaierror:
        raise usage.UsageError("failed to resolve %s to an address" % value)


def convert_option_ston(key, value):
    # special "infinite" value reserved for some flags
    if (isinstance(value, basestring) and key == "http-max-retries"
            and value.lower() in ("inf", "infinite", "unlimited")):
        return float("inf")

    try:
        return convert.ston(value)

    except ValueError:
        raise usage.UsageError(
            "--%s requires a number but got %s" % (key, repr(value)))


def convert_option_projects(key, value):
    return filter(bool, map(str.strip, value.split(",")))


class Options(usage.Options):
    optParameters = [
        # local 'server' settings
        ("port", "p", 50000,
         "The port which the master should use to talk back to the agent."),

        # agent -> master communications
        ("master", "", None,
         "The master server's hostname or address.  If no other options are "
         "provided to describe the location of specific services then the "
         "resource urls will be built off of this address."),
        ("http-auth-user", "", None,
         "the user to use for connecting to the master's REST api.  The "
         "default is communication without authentication."),
        ("http-auth-password", "", None,
         "the password to use to connect to the master's REST api.  The "
         "default is communication without authentication."),
        ("http-api-port", "", 5000,
         "Default port the restfull HTTP api runs on.  By default this value "
         "is combined with --master but could also be combined with the "
         "alternate --http-api-server"),
        ("http-api-scheme", "", "http",
         "The scheme to use to communicate over http.  Valid values are "
         "'http' or 'https'"),

        # statsd setup
        ("statsd-port", "", 8125,
         "Default port that statsd runs on.  By default this value is combined "
         "with --master but could also be combined with the alternate "
         "--statsd-server"),
        ("http-api-server", "", None,
         "Alternate server which is running the restfull HTTP server.  This "
         "will replace the value provided by the --master flag."),
        ("statsd-server", "", None,
         "Alternate server which is running the statsd service.  This will "
         "replace the value provided by the --master flag."),

        # http retries/detailed configuration
        ("http-max-retries", "", "unlimited",
         "the max number of times to retry a request back to the master"),
        ("http-retry-delay", "", 3,
         "if a http request back to the master has failed, wait this amount of "
         "time before trying again"),

        # local agent settings which control some resources
        # and restrictions
        ("memory-check-interval", "", 10,
         "how often swap and ram resources should be checked and sent to the "
         "master"),
        ("projects", "", "*",
         "A comma separated list of projects this agent is allowed to do work "
         "for.  Note however that this only updates the master at the time "
         "the agent is run.  Once the agent has been launched all further "
         "'membership' is present in the database and acted on by the queue.")]

    # special variable used for inferring type in makeService
    optConverters = {
        "port": convert_option_ston,
        "memory-check-interval": convert_option_ston,
        "http-max-retries": convert_option_ston,
        "http-retry-delay": convert_option_ston,
        "http-api-port": convert_option_ston,
        "statsd-port": convert_option_ston,
        "projects": convert_option_projects}
#
#class IPCReceieverFactory(Factory):
#    """
#    Receives incoming connections and hands them off to the protocol
#    object.  In addition this class will also keep a list of all hosts
#    which have connected so they can be notified upon shutdown.
#    """
#    protocol = IPCProtocol
#    known_hosts = set()
#
#    def stopFactory(self):  # TODO: notify all connected hosts we are stopping
#        if self.known_hosts:
{'fraction_non_alphanumeric': 0.09182844243792325, 'fraction_numerical': 0.0043340857787810385, 'mean_word_length': 2.848505906879778, 'pattern_counts': {'":': 13, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6622941', 'n_tokens_mistral': 2955, 'n_tokens_neox': 2873, 'n_words': 1229}