<commit_before>#!/usr/bin/python
# encoding: utf-8
#
# Copyright 2014 The Pennsylvania State University.
#
"""
BESRelevanceProvider.py

Created by <NAME> (<EMAIL>) on 2014-02-19.

AutoPkg Processor for retreiving relevance data for tasks.
"""

import os
import hashlib
import sys
import subprocess

from autopkglib import Processor, ProcessorError

__all__ = ["BESRelevanceProvider"]

QNA = '/Applications/Utilities/QnA.app/Contents/Resources/QnA'

class BESRelevanceProvider(Processor):
    description = "Retreives arbitrary relevance data using the QnA utility."
    input_variables = {
        "bes_filepath": {
            "required": True,
            "description":
                "Path to a file for relevance data. Defaults to %pathname%."
        },
        "bes_relevance": {
            "required": False,
            "description":
                "A line of relevance to evaluate in QnA and return the result."
        },
    }
    output_variables = {
        "bes_sha1": {
            "description":
                "The resulting file sha1 of the %bes_filepath%."
        },
        "bes_sha1_short": {
            "description":
                "The short end of %bes_sha1% to be used as a display version."
        },
        "bes_size": {
            "description":
                "The resulting file size of the %bes_filepath%."
        },
        "bes_relevance_result": {
            "description":
                "The result of the relevance expression, if provided."
        },
    }
    __doc__ = description

    def eval_relevance(self, relevance):
        try:
            proc = subprocess.Popen(QNA, bufsize=-1,
                                    stdin=subprocess.PIPE,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
            out, err = proc.communicate(relevance)

            output = {}
            for line in out.strip().split('\n'):
                output[line.split(':')[0].strip()] = line.split(':')[1].strip()

            if output.get('E', None):
                self.output("Relevance Error: {%s} -- %s" %
                            (relevance, output.get('E')))
                return None
            else:
                return output.get('A', None)
        except Exception, error:
            self.output("QnA Error: (%s) -- %s" % (QNA, error))
            return None

    def main(self):
        # Assign BES Console Variables
        bes_filepath = self.env.get("bes_filepath", self.env.get("pathname"))
        bes_relevance = self.env.get("bes_relevance", None)

        if os.path.isfile(bes_filepath):

            self.env['bes_sha1'] = hashlib.sha1(file(
                bes_filepath).read()).hexdigest()
            self.env['bes_size'] = str(os.path.getsize(bes_filepath))
            self.env['bes_sha1_short'] = str(self.env.get("bes_sha1"))[-5:]

        self.output("bes_sha1 = %s, bes_size = %s, bes_sha1_short = %s" %
                    (self.env.get("bes_sha1"),
                     self.env.get("bes_size"),
                     self.env.get("bes_sha1_short")))

        if bes_relevance:
            self.env['bes_relevance_result'] = self.eval_relevance(bes_relevance)

if __name__ == "__main__":
    processor = BESRelevanceProvider()
    processor.execute_shell()<commit_msg>Updated BESRelevanceProvider with ability to specify output variable names<commit_after>#!/usr/bin/python
# encoding: utf-8
#
# Copyright 2014 The Pennsylvania State University.
#
"""
BESRelevanceProvider.py

Created by <NAME> (<EMAIL>) on 2014-02-19.

AutoPkg Processor for retreiving relevance data for tasks.
"""

import os
import hashlib
import subprocess

from autopkglib import Processor, ProcessorError

__all__ = ["BESRelevanceProvider"]

QNA = '/Applications/Utilities/QnA.app/Contents/Resources/QnA'

class BESRelevanceProvider(Processor):
    description = "Retreives arbitrary relevance data using the QnA utility."
    input_variables = {
        "bes_filepath": {
            "required": True,
            "description":
                "Path to a file for relevance data. Defaults to %pathname%."
        },
        "bes_relevance": {
            "required": False,
            "description":
                "A line of relevance to evaluate in QnA and return the result."
        },
        "output_var_name": {
            "required": False,
            "description":
                "Output variable name. Defaults to 'bes_relevance_result'"
        },
    }
    output_variables = {
        "bes_sha1": {
            "description":
                "The resulting file sha1 of the %bes_filepath%."
        },
        "bes_sha1_short": {
            "description":
                "The short end of %bes_sha1% to be used as a display version."
        },
        "bes_size": {
            "description":
                "The resulting file size of the %bes_filepath%."
        },
    }
    __doc__ = description

    def eval_relevance(self, relevance):
        # Evaluate Relevance Expression
        try:
            proc = subprocess.Popen(QNA, bufsize=-1,
                                    stdin=subprocess.PIPE,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
            out, err = proc.communicate(relevance)

            output = {}
            for line in out.strip().split('\n'):
                output[line.split(':')[0].strip()] = line.split(':')[1].strip()

            if output.get('E', None):
                self.output("Relevance Error: {%s} -- %s" %
                            (relevance, output.get('E')))
                return None
            else:
                return output.get('A', None)
        except Exception, error:
            self.output("QnA Error: (%s) -- %s" % (QNA, error))
            return None

    def main(self):
        # Assign BES Console Variables
        bes_filepath = self.env.get("bes_filepath", self.env.get("pathname"))
        bes_relevance = self.env.get("bes_relevance", None)

        if os.path.isfile(bes_filepath):

            self.env['bes_sha1'] = hashlib.sha1(file(
                bes_filepath).read()).hexdigest()
            self.env['bes_size'] = str(os.path.getsize(bes_filepath))
            self.env['bes_sha1_short'] = str(self.env.get("bes_sha1"))[-5:]

        self.output("bes_sha1 = %s, bes_size = %s, bes_sha1_short = %s" %
                    (self.env.get("bes_sha1"),
                     self.env.get("bes_size"),
                     self.env.get("bes_sha1_short")))

        if bes_relevance:
            output_var_name = self.env.get("output_var_name", 
                                           "bes_relevance_result")

            self.env[output_var_name] = self.eval_relevance(bes_relevance)
            self.output("%s = %s" %
                        (output_var_name,
                         self.env.get(output_var_name)))

if __name__ == "__main__":
    processor = BESRelevanceProvider()
    processor.execute_shell()
{'fraction_non_alphanumeric': 0.11905100156272198, 'fraction_numerical': 0.00823980679073732, 'mean_word_length': 1.8137490007993604, 'pattern_counts': {'":': 31, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1525691', 'n_tokens_mistral': 2137, 'n_tokens_neox': 2007, 'n_words': 443}