<commit_before># No shebang line, this module is meant to be imported
#
# Copyright 2014 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from decimal import Decimal

try:
    from httplib import ACCEPTED, BAD_REQUEST, CONFLICT, SERVICE_UNAVAILABLE
except ImportError:  # pragma: no cover
    from http.client import ACCEPTED, BAD_REQUEST, CONFLICT, SERVICE_UNAVAILABLE

from twisted.web.server import NOT_DONE_YET
from voluptuous import Invalid, Schema, Required, Optional, Any

from pyfarm.core.enums import STRING_TYPES
from pyfarm.agent.config import config
from pyfarm.agent.http.api.base import APIResource
from pyfarm.agent.logger import getLogger
from pyfarm.agent.utility import request_from_master
from pyfarm.agent.sysinfo.memory import ram_free
from pyfarm.jobtypes.core.jobtype import JobType

logger = getLogger("agent.http.assign")

# Values used by the schema to do type testing
# of input requests
STRINGS = Any(*STRING_TYPES)
try:
    WHOLE_NUMBERS = Any(*(int, long))
    NUMBERS = Any(*(int, long, float, Decimal))
except NameError:  # pragma: no cover
    WHOLE_NUMBERS = int
    NUMBERS = Any(*(int, float, Decimal))


def validate_environment(values):
    """
    Ensures that ``values`` is a dictionary and that it only
    contains string keys and values.
    """
    if not isinstance(values, dict):
        raise Invalid("Expected a dictionary")

    for key, value in values.items():
        if not isinstance(key, STRING_TYPES):
            raise Invalid("Key %r must be a string" % key)

        if not isinstance(value, STRING_TYPES):
            raise Invalid("Value %r for key %r must be a string" % (key, value))


class HandlePost(object):
    def __init__(self, data):
        self.data = data


class Assign(APIResource):
    isLeaf = False  # this is not really a collection of things

    # Schemas used for validating the request before
    # the target function will handle it.  These make
    # assertions about what kind of input data is required
    # or not based on the agent's internal code.
    SCHEMAS = {
        "POST": Schema({
            Required("job"): Schema({
                Required("id"): WHOLE_NUMBERS,
                Required("by"): NUMBERS,
                Optional("batch"): WHOLE_NUMBERS,
                Optional("user"): STRINGS,
                Optional("ram"): WHOLE_NUMBERS,
                Optional("ram_warning"): WHOLE_NUMBERS,
                Optional("ram_max"): WHOLE_NUMBERS,
                Optional("cpus"): WHOLE_NUMBERS,
                Optional("data"): dict,
                Optional("environ"): validate_environment,
                Optional("title"): STRINGS}),
            Required("jobtype"): {
                Required("name"): STRINGS,
                Required("version"): WHOLE_NUMBERS},
            Required("tasks"): lambda values: map(
                Schema({
                    Required("id"): WHOLE_NUMBERS,
                    Required("frame"): NUMBERS}), values)})}

    def post(self, **kwargs):
        request = kwargs["request"]
        data = kwargs["data"]

        if request_from_master(request):
            config.master_contacted()

        # First, get the resources we have *right now*.  In some cases
        # this means using the functions in pyfarm.core.sysinfo because
        # entries in `config` could be slightly out of sync with the system.
        memory_free = int(ram_free())
        cpus = config["cpus"]
        requires_ram = data["job"].get("ram")
        requires_cpus = data["job"].get("cpus")

        if "restart_requested" in config and config["restart_requested"] == True:
            logger.error("Rejecting assignment because of scheduled restart.")
            request.setResponseCode(SERVICE_UNAVAILABLE)
            request.write(
                {"error": "Agent cannot accept assignments because of a "
                          "pending restart"})
            request.finish()
            return NOT_DONE_YET

        if "agent-id" not in config:
            logger.error(
                "Agent has not yet connected to the master or `agent-id` "
                "has not been set yet.")
            request.setResponseCode(BAD_REQUEST)
            request.write(
                {"error": "agent-id has not been set in the config"})
            request.finish()
            return NOT_DONE_YET<commit_msg>== True -> is True<commit_after># No shebang line, this module is meant to be imported
#
# Copyright 2014 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from decimal import Decimal

try:
    from httplib import ACCEPTED, BAD_REQUEST, CONFLICT, SERVICE_UNAVAILABLE
except ImportError:  # pragma: no cover
    from http.client import ACCEPTED, BAD_REQUEST, CONFLICT, SERVICE_UNAVAILABLE

from twisted.web.server import NOT_DONE_YET
from voluptuous import Invalid, Schema, Required, Optional, Any

from pyfarm.core.enums import STRING_TYPES
from pyfarm.agent.config import config
from pyfarm.agent.http.api.base import APIResource
from pyfarm.agent.logger import getLogger
from pyfarm.agent.utility import request_from_master
from pyfarm.agent.sysinfo.memory import ram_free
from pyfarm.jobtypes.core.jobtype import JobType

logger = getLogger("agent.http.assign")

# Values used by the schema to do type testing
# of input requests
STRINGS = Any(*STRING_TYPES)
try:
    WHOLE_NUMBERS = Any(*(int, long))
    NUMBERS = Any(*(int, long, float, Decimal))
except NameError:  # pragma: no cover
    WHOLE_NUMBERS = int
    NUMBERS = Any(*(int, float, Decimal))


def validate_environment(values):
    """
    Ensures that ``values`` is a dictionary and that it only
    contains string keys and values.
    """
    if not isinstance(values, dict):
        raise Invalid("Expected a dictionary")

    for key, value in values.items():
        if not isinstance(key, STRING_TYPES):
            raise Invalid("Key %r must be a string" % key)

        if not isinstance(value, STRING_TYPES):
            raise Invalid("Value %r for key %r must be a string" % (key, value))


class HandlePost(object):
    def __init__(self, data):
        self.data = data


class Assign(APIResource):
    isLeaf = False  # this is not really a collection of things

    # Schemas used for validating the request before
    # the target function will handle it.  These make
    # assertions about what kind of input data is required
    # or not based on the agent's internal code.
    SCHEMAS = {
        "POST": Schema({
            Required("job"): Schema({
                Required("id"): WHOLE_NUMBERS,
                Required("by"): NUMBERS,
                Optional("batch"): WHOLE_NUMBERS,
                Optional("user"): STRINGS,
                Optional("ram"): WHOLE_NUMBERS,
                Optional("ram_warning"): WHOLE_NUMBERS,
                Optional("ram_max"): WHOLE_NUMBERS,
                Optional("cpus"): WHOLE_NUMBERS,
                Optional("data"): dict,
                Optional("environ"): validate_environment,
                Optional("title"): STRINGS}),
            Required("jobtype"): {
                Required("name"): STRINGS,
                Required("version"): WHOLE_NUMBERS},
            Required("tasks"): lambda values: map(
                Schema({
                    Required("id"): WHOLE_NUMBERS,
                    Required("frame"): NUMBERS}), values)})}

    def post(self, **kwargs):
        request = kwargs["request"]
        data = kwargs["data"]

        if request_from_master(request):
            config.master_contacted()

        # First, get the resources we have *right now*.  In some cases
        # this means using the functions in pyfarm.core.sysinfo because
        # entries in `config` could be slightly out of sync with the system.
        memory_free = int(ram_free())
        cpus = config["cpus"]
        requires_ram = data["job"].get("ram")
        requires_cpus = data["job"].get("cpus")

        if "restart_requested" in config and config["restart_requested"] is True:
            logger.error("Rejecting assignment because of scheduled restart.")
            request.setResponseCode(SERVICE_UNAVAILABLE)
            request.write(
                {"error": "Agent cannot accept assignments because of a "
                          "pending restart"})
            request.finish()
            return NOT_DONE_YET

        if "agent-id" not in config:
            logger.error(
                "Agent has not yet connected to the master or `agent-id` "
                "has not been set yet.")
            request.setResponseCode(BAD_REQUEST)
            request.write(
                {"error": "agent-id has not been set in the config"})
            request.finish()
            return NOT_DONE_YET
{'fraction_non_alphanumeric': 0.09301368453544603, 'fraction_numerical': 0.0016462599032822307, 'mean_word_length': 2.5814296241709656, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2207455', 'n_tokens_mistral': 2705, 'n_tokens_neox': 2658, 'n_words': 915}