// Path: Hadoop_Summit_Pig_for_Data_Science/src/main/java/com/caseystella/ds/nlp/SentimentAnalyzer.java
// public enum SentimentAnalyzer implements Function<String, SentimentClass> {
//     INSTANCE;
// 
//     @Override
//     public SentimentClass apply(String document)
//     {
//         // shut off the annoying intialization messages
//         Properties props = new Properties();
//         //specify the annotators that we want to use to annotate the text.  We need a tokenized sentence with POS tags to extract sentiment.
//         //this forms our pipeline
//         props.setProperty("annotators", "tokenize, ssplit, parse, sentiment");
//         StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
//         Annotation annotation = pipeline.process(document);
//         List<Sentence> sentences = new ArrayList<Sentence>();
//         /*
//          * We're going to iterate over all of the sentences and extract the sentiment.  We'll adopt a majority rule policy
//          */
//         for( CoreMap sentence : annotation.get(CoreAnnotations.SentencesAnnotation.class))
//         {
//             //for each sentence, we get the sentiment that CoreNLP thinks this sentence indicates.
//             Tree sentimentTree = sentence.get(SentimentCoreAnnotations.AnnotatedTree.class);
//             int sentimentClassIdx = RNNCoreAnnotations.getPredictedClass(sentimentTree);
//             SentimentClass sentimentClass = SentimentClass.getSpecific(sentimentClassIdx);
// 
//             /*
//              * Each possible sentiment has an associated probability, so let's pull the entire
//              * set of probabilities across all sentiment classes.
//              */
//             double[] probs = new double[SentimentClass.values().length];
//             {
//                 SimpleMatrix mat = RNNCoreAnnotations.getPredictions(sentimentTree);
//                 for(int i = 0;i < SentimentClass.values().length;++i)
//                 {
//                     probs[i] = mat.get(i);
//                 }
//             }
//             /*
//              * Add the sentence and the associated probabilities to our list.
//              */
//             String sentenceStr = AnnotationUtils.sentenceToString(sentence).replace("\n", "");
//             sentences.add(new Sentence(probs, sentenceStr, sentimentClass));
//         }
// 
//         /*
//          * Finally, rollup the score of the entire document given the list of sentiments
//          * for each sentence.
//          *
//          * I have found that this is a surprisingly difficult thing.  We're going to use the fact that
//          * we are mapping these documents onto Positive, Negative to use a technique that people use
//          * to figure out overall product ratings from individual star ratings, the Wilson Score.
//          * See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html.
//          *
//          * For your convenience, I have implemented a number of other approaches, including:
//          *  * Basic majority-rule voting
//          *  * Max probability across all sentiments for all sentences
//          *  * Last sentence wins
//          *  * Longest sentence wins
//          *
//          */
//         return SentimentRollup.WILSON_SCORE.apply(sentences);
//     }
// }
// 
// Path: Hadoop_Summit_Pig_for_Data_Science/src/main/java/com/caseystella/ds/nlp/SentimentClass.java
// public enum SentimentClass {
//     VERY_NEGATIVE(0)
//    , NEGATIVE(1)
//    , NEUTRAL(2)
//    , POSITIVE(3)
//    , VERY_POSITIVE(4)
//     ;
//     private int index = -1;
//     SentimentClass(int idx)
//     {
//         index = idx;
//     }
//     public int getIndex() { return index;}
//     public static SentimentClass getSpecific(int sentimentClass)
//     {
//         return SentimentClass.values()[sentimentClass];
//     }
// 
//     public static SentimentClass getGeneral(int sentimentClass)
//     {
//         //remove the extreme cases
//         if(sentimentClass == 0) sentimentClass = 1;
//         if(sentimentClass == 4) sentimentClass = 3;
//         return SentimentClass.values()[sentimentClass];
//     }
// }


import com.caseystella.ds.nlp.SentimentAnalyzer;
import com.caseystella.ds.nlp.SentimentClass;
import edu.stanford.nlp.util.logging.RedwoodConfiguration;
import org.apache.pig.EvalFunc;
import org.apache.pig.data.Tuple;
import java.io.IOException;
package com.caseystella.ds.pig;



/**
 * A pig UDF which accepts a document and returns a sentiment class: Positive or Negative
 * The assumptions are that this document is a movie review and uses, under the hood, the
 * Stanford CoreNLP Sentiment classifier.
 */
public class ANALYZE_SENTIMENT extends EvalFunc<String>
{
    @Override
    public String exec(Tuple objects) throws IOException
    {
        //stop CoreNLP from printing to stdout
        RedwoodConfiguration.empty().capture(System.err).apply();
        //grab the document
        String document = (String)objects.get(0);
        if(document == null || document.length() == 0)
        {
            //if the document is malformed, return null, which is to say, we don't know how to
            //handle it
            return null;
        }
        //Call out to our handler that we wrote to do the sentiment analysis
// Path: Hadoop_Summit_Pig_for_Data_Science/src/main/java/com/caseystella/ds/nlp/SentimentAnalyzer.java
// public enum SentimentAnalyzer implements Function<String, SentimentClass> {
//     INSTANCE;
// 
//     @Override
//     public SentimentClass apply(String document)
//     {
//         // shut off the annoying intialization messages
//         Properties props = new Properties();
//         //specify the annotators that we want to use to annotate the text.  We need a tokenized sentence with POS tags to extract sentiment.
//         //this forms our pipeline
//         props.setProperty("annotators", "tokenize, ssplit, parse, sentiment");
//         StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
//         Annotation annotation = pipeline.process(document);
//         List<Sentence> sentences = new ArrayList<Sentence>();
//         /*
//          * We're going to iterate over all of the sentences and extract the sentiment.  We'll adopt a majority rule policy
//          */
//         for( CoreMap sentence : annotation.get(CoreAnnotations.SentencesAnnotation.class))
//         {
//             //for each sentence, we get the sentiment that CoreNLP thinks this sentence indicates.
//             Tree sentimentTree = sentence.get(SentimentCoreAnnotations.AnnotatedTree.class);
//             int sentimentClassIdx = RNNCoreAnnotations.getPredictedClass(sentimentTree);
//             SentimentClass sentimentClass = SentimentClass.getSpecific(sentimentClassIdx);
// 
//             /*
//              * Each possible sentiment has an associated probability, so let's pull the entire
//              * set of probabilities across all sentiment classes.
//              */
//             double[] probs = new double[SentimentClass.values().length];
//             {
//                 SimpleMatrix mat = RNNCoreAnnotations.getPredictions(sentimentTree);
//                 for(int i = 0;i < SentimentClass.values().length;++i)
//                 {
//                     probs[i] = mat.get(i);
//                 }
//             }
//             /*
//              * Add the sentence and the associated probabilities to our list.
//              */
//             String sentenceStr = AnnotationUtils.sentenceToString(sentence).replace("\n", "");
//             sentences.add(new Sentence(probs, sentenceStr, sentimentClass));
//         }
// 
//         /*
//          * Finally, rollup the score of the entire document given the list of sentiments
//          * for each sentence.
//          *
//          * I have found that this is a surprisingly difficult thing.  We're going to use the fact that
//          * we are mapping these documents onto Positive, Negative to use a technique that people use
//          * to figure out overall product ratings from individual star ratings, the Wilson Score.
//          * See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html.
//          *
//          * For your convenience, I have implemented a number of other approaches, including:
//          *  * Basic majority-rule voting
//          *  * Max probability across all sentiments for all sentences
//          *  * Last sentence wins
//          *  * Longest sentence wins
//          *
//          */
//         return SentimentRollup.WILSON_SCORE.apply(sentences);
//     }
// }
// 
// Path: Hadoop_Summit_Pig_for_Data_Science/src/main/java/com/caseystella/ds/nlp/SentimentClass.java
// public enum SentimentClass {
//     VERY_NEGATIVE(0)
//    , NEGATIVE(1)
//    , NEUTRAL(2)
//    , POSITIVE(3)
//    , VERY_POSITIVE(4)
//     ;
//     private int index = -1;
//     SentimentClass(int idx)
//     {
//         index = idx;
//     }
//     public int getIndex() { return index;}
//     public static SentimentClass getSpecific(int sentimentClass)
//     {
//         return SentimentClass.values()[sentimentClass];
//     }
// 
//     public static SentimentClass getGeneral(int sentimentClass)
//     {
//         //remove the extreme cases
//         if(sentimentClass == 0) sentimentClass = 1;
//         if(sentimentClass == 4) sentimentClass = 3;
//         return SentimentClass.values()[sentimentClass];
//     }
// }


// Path: Hadoop_Summit_Pig_for_Data_Science/src/main/java/com/caseystella/ds/pig/ANALYZE_SENTIMENT.java
import com.caseystella.ds.nlp.SentimentAnalyzer;
import com.caseystella.ds.nlp.SentimentClass;
import edu.stanford.nlp.util.logging.RedwoodConfiguration;
import org.apache.pig.EvalFunc;
import org.apache.pig.data.Tuple;
import java.io.IOException;

package com.caseystella.ds.pig;



/**
 * A pig UDF which accepts a document and returns a sentiment class: Positive or Negative
 * The assumptions are that this document is a movie review and uses, under the hood, the
 * Stanford CoreNLP Sentiment classifier.
 */
public class ANALYZE_SENTIMENT extends EvalFunc<String>
{
    @Override
    public String exec(Tuple objects) throws IOException
    {
        //stop CoreNLP from printing to stdout
        RedwoodConfiguration.empty().capture(System.err).apply();
        //grab the document
        String document = (String)objects.get(0);
        if(document == null || document.length() == 0)
        {
            //if the document is malformed, return null, which is to say, we don't know how to
            //handle it
            return null;
        }
        //Call out to our handler that we wrote to do the sentiment analysis
        SentimentClass sentimentClass = SentimentAnalyzer.INSTANCE.apply(document);