<commit_before># BEGIN_COPYRIGHT
#
# Copyright 2009-2014 CRS4.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# END_COPYRIGHT

"""
Pydoop API
==========

All computation is performed by instances of classes derived from Mapper,
Reducer and, possibly, Reader, Writer, Combiner and Partitioner.

All communication between the framework and these objects is mediated by
Context objects. That is:

 * when they are instantiated, they receive contextual
   information via a context object::

     def __init__(self, context):
         job_conf = context.job_conf
         input_split = context.input_split

 * when they are used, for instance invoking the map method of a mapper,
   they read data and emit back to the framework via a context object::

     def map(self, context):
         key, value = context.key, context.value
         ...
         context.emit(new_key, new_value)
"""

from abc import ABCMeta, abstractmethod


class PydoopError(Exception):
    pass


class Counter(object):
    """
    An interface to the Hadoop counters infrastructure.
    """
    def __init__(self, counter_id):
        self.id = counter_id

    def get_id(self):
        return self.id


class JobConf(dict):
    """
    Configuration properties assigned to this job.
    """
    def __init__(self, values):
        if 1 & len(values):
            raise PydoopError('JobConf.__init__: len(values) should be even')
        super(JobConf, self).__init__(zip(values[::2], values[1::2]))

    def get_int(self, key, default=None):
        return int(self.get(key, default))

    def get_float(self, key, default=None):
        return float(self.get(key, default))

    def get_bool(self, key, default=None):
        return bool(self.get(key, default))


class Context(object):
    """
    Information inter-exchange object.

    Context instances are the only points of contact between the framework
    and user-defined code.
    """
    __metaclass__ = ABCMeta

    @abstractmethod
    def get_job_conf(self):
        pass

    @abstractmethod
    def get_input_key(self):
        pass

    @abstractmethod
    def get_input_value(self):
        pass

    @abstractmethod
    def emit(self, key, value):
        pass

    @abstractmethod
    def progress(self):
        pass

    @abstractmethod
    def set_status(self, status):
        pass

    @abstractmethod
    def get_counter(self, group, name):
        pass

    @abstractmethod
    def increment_counter(self, counter, amount):
        pass


class MapContext(Context):

    @abstractmethod
    def get_input_split(self):
        pass

    @abstractmethod
    def get_input_key_class(self):
        pass

    @abstractmethod
    def get_input_value_class(self):
        pass


class ReduceContext(Context):

    @abstractmethod
    def next_value(self):
        pass


class Closable(object):
    def close(self):
        pass<commit_msg>Context (and derived classes) attributes are explicitly advertised in api.<commit_after># BEGIN_COPYRIGHT
#
# Copyright 2009-2014 CRS4.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# END_COPYRIGHT

"""
Pydoop API
==========

All computation is performed by instances of classes derived from Mapper,
Reducer and, possibly, Reader, Writer, Combiner and Partitioner.

All communication between the framework and these objects is mediated by
Context objects. That is:

 * when they are instantiated, they receive contextual
   information via a context object::

     def __init__(self, context):
         job_conf = context.job_conf
         input_split = context.input_split

 * when they are used, for instance invoking the map method of a mapper,
   they read data and emit back to the framework via a context object::

     def map(self, context):
         key, value = context.key, context.value
         ...
         context.emit(new_key, new_value)
"""

from abc import ABCMeta, abstractmethod


class PydoopError(Exception):
    pass


class Counter(object):
    """
    An interface to the Hadoop counters infrastructure.
    """
    def __init__(self, counter_id):
        self.id = counter_id

    def get_id(self):
        return self.id


class JobConf(dict):
    """
    Configuration properties assigned to this job.
    """
    def __init__(self, values):
        if 1 & len(values):
            raise PydoopError('JobConf.__init__: len(values) should be even')
        super(JobConf, self).__init__(zip(values[::2], values[1::2]))

    def get_int(self, key, default=None):
        return int(self.get(key, default))

    def get_float(self, key, default=None):
        return float(self.get(key, default))

    def get_bool(self, key, default=None):
        return bool(self.get(key, default))


class Context(object):
    """
    Information inter-exchange object.

    Context instances are the only points of contact between the framework
    and user-defined code.
    """
    __metaclass__ = ABCMeta

    @property
    def job_conf(self):
        return self.get_job_conf()

    @abstractmethod
    def get_job_conf(self):
        pass

    @property
    def key(self):
        return self.get_input_key()
    
    @abstractmethod
    def get_input_key(self):
        pass

    @property
    def value(self):
        return self.get_input_value()
    
    @abstractmethod
    def get_input_value(self):
        pass

    @abstractmethod
    def emit(self, key, value):
        pass

    @abstractmethod
    def progress(self):
        pass

    @abstractmethod
    def set_status(self, status):
        pass

    @abstractmethod
    def get_counter(self, group, name):
        pass

    @abstractmethod
    def increment_counter(self, counter, amount):
        pass


class MapContext(Context):

    @property
    def input_split(self):
        return self.get_input_split()

    @abstractmethod
    def get_input_split(self):
        pass

    @property
    def input_key_class(self):
        return self.get_input_key_class()
    
    @abstractmethod
    def get_input_key_class(self):
        pass

    @property
    def input_value_class(self):
        return self.get_input_value_class()
        
    @abstractmethod
    def get_input_value_class(self):
        pass


class ReduceContext(Context):

    @property
    def values(self):
        return self.get_input_values()
    
    @abstractmethod
    def get_input_values(self):
        pass
    
    @abstractmethod
    def next_value(self):
        pass


class Closable(object):
    def close(self):
        pass
{'fraction_non_alphanumeric': 0.0842358604091456, 'fraction_numerical': 0.00454606230779516, 'mean_word_length': 2.8897555902236087, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7347226', 'n_tokens_mistral': 2172, 'n_tokens_neox': 2072, 'n_words': 705}