<commit_before>import commandline.hadoop
import spark.bootstrap_spark
import os


class PilotComputeDescription(dict):
    """ B{PilotComputeDescription (PCD).}

        A PilotComputeDescription is a based on the attributes defined on
        the SAGA Job Description.

        The PilotComputeDescription is used by the application to specify
        what kind of PilotJobs it requires.

        Example::
             pilot_compute_description = {
                           resource_url="fork://localhost",
                           working_directory="/tmp",
                           number_cores=1,
                           cores_per_node=1
                           type="spark"
                           }

        B{Attention}: The PilotComputeDescription is mapped 1:1 to the underlying SAGA-Python
        job description, which is used for launching the pilot that manages the Hadoop cluster.

    """

    def __init__(self):
        pass


    def __setattr__(self, attr, value):
        self[attr]=value


    def __getattr__(self, attr):
        return self[attr]


    #############################################################################

class PilotCompute(object):
    """ B{PilotCompute (PC)}.

        This is the object that is returned by the PilotComputeService when a
        new PilotCompute (aka Pilot-Job) is created based on a PilotComputeDescription.

        A Pilot-Compute in Pilot-Hadoop represents an active Hadoop YARN or Spark cluster

        The PilotCompute object can be used by the application to keep track
        of PilotComputes that are active.

        A PilotCompute has state, can be queried, can be cancelled and be
        re-initialized.
    """



    def __init__(self, saga_job, details):
        self.saga_job = saga_job
        self.details = details


    def cancel(self):
        """ Remove the PilotCompute from the PilotCompute Service.

            Keyword arguments:
            None
        """
        self.saga_job.cancel()


    def get_state(self):
        self.saga_job.get_state()

<commit_msg>refinement of API<commit_after>#######################
# Mac OS:
# brew install apache-spark
# SPARK_HOME='/usr/local/Cellar/apache-spark/1.3.0/libexec/'
# Start Spark: /usr/local/Cellar/apache-spark/1.3.0/libexec/sbin/start-all.sh
# py4j needs be installed in your virtualenv

import commandline.hadoop
import spark.bootstrap_spark
import os, sys

spark_home='/usr/local/Cellar/apache-spark/1.3.0/libexec/'
#if details!=None:
#    spark_home = details["spark_home"]
os.environ["SPARK_HOME"] = spark_home
sys.path.insert(0, os.path.join(spark_home, "python"))

# import Spark Libraries
from pyspark import SparkContext, SparkConf, Accumulator, AccumulatorParam
from pyspark.sql import SQLContext
from pyspark.sql.types import *
from pyspark.mllib.linalg import Vector


class PilotComputeDescription(dict):
    """ B{PilotComputeDescription (PCD).}

        A PilotComputeDescription is a based on the attributes defined on
        the SAGA Job Description.

        The PilotComputeDescription is used by the application to specify
        what kind of PilotJobs it requires.

        Example::
             pilot_compute_description = {
                           resource_url="fork://localhost",
                           working_directory="/tmp",
                           number_cores=1,
                           cores_per_node=1
                           type="spark"
                           }

        B{Attention}: The PilotComputeDescription is mapped 1:1 to the underlying SAGA-Python
        job description, which is used for launching the pilot that manages the Hadoop cluster.

    """

    def __init__(self):
        pass


    def __setattr__(self, attr, value):
        self[attr]=value


    def __getattr__(self, attr):
        return self[attr]


    #############################################################################

class PilotCompute(object):
    """ B{PilotCompute (PC)}.

        This is the object that is returned by the PilotComputeService when a
        new PilotCompute (aka Pilot-Job) is created based on a PilotComputeDescription.

        A Pilot-Compute in Pilot-Hadoop represents an active Hadoop YARN or Spark cluster

        The PilotCompute object can be used by the application to keep track
        of PilotComputes that are active.

        A PilotCompute has state, can be queried, can be cancelled and be
        re-initialized.
    """



    def __init__(self, saga_job, details):
        self.saga_job = saga_job
        self.details = details
        self.spark_context = None

    def cancel(self):
        """ Remove the PilotCompute from the PilotCompute Service.

            Keyword arguments:
            None
        """
        self.saga_job.cancel()


    def get_state(self):
        self.saga_job.get_state()


    def get_spark_context(self):
        if self.spark_context == None:
            self.spark_context = SparkContext(self.details["master_url"],
                                              "Pilot-Spark",
                                              sparkHome=spark_home)
        return self.spark_context

{'fraction_non_alphanumeric': 0.10345493148040918, 'fraction_numerical': 0.003667245705462266, 'mean_word_length': 2.23875, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '628727', 'n_tokens_mistral': 1436, 'n_tokens_neox': 1322, 'n_words': 414}