<commit_before>/*
 * (C) 2010-2012 ICM UW. All rights reserved.
 */
package pl.edu.icm.coansys.disambiguation.author.jobs.hdfs;

import java.lang.management.ManagementFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.mapreduce.TableInputFormat;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.log4j.Logger;
import pl.edu.icm.coansys.disambiguation.auxil.LoggingInDisambiguation;
import pl.edu.icm.coansys.disambiguation.auxil.TextTextArrayMapWritable;
import pl.edu.icm.coansys.importers.constants.HBaseConstant;

/**
 *
 * @author pdendek
 * @version 1.0
 * @since 2012-08-07
 */
public class DisambiguationJob_Toy extends Configured implements Tool {

    /*
     * 
     * Inner fields
     * 
     */
    private static Logger logger = Logger.getLogger(LoggingInDisambiguation.class);
    private Configuration conf;
    /*
     * 
     * Fields to be set
     * 
     */
    String INPUT_TABLE = null;
    String OUTPUT_DIR = null;
    String FEATURES_DESCRIPTION = null;
    String NAME = null;
    String THRESHOLD = null;
    int REDUCER_NUM = 65;

    /*
     * 
     * Getters and setters
     * 
     */
    public String getINPUT_TABLE() {
        return INPUT_TABLE;
    }

    public DisambiguationJob_Toy setINPUT_TABLE(String iNPUT_TABLE) {
        INPUT_TABLE = iNPUT_TABLE;
        return this;
    }

    public String getOUTPUT_DIR() {
        return OUTPUT_DIR;
    }

    public DisambiguationJob_Toy setOUTPUT_TABLE(String oUTPUT_TABLE) {
        OUTPUT_DIR = oUTPUT_TABLE;
        return this;
    }

    public String getNAME() {
        return NAME;
    }

    public DisambiguationJob_Toy setNAME(String nAME) {
        NAME = nAME;
        return this;
    }

    public String getFEATURES_DESCRIPTION() {
        return FEATURES_DESCRIPTION;
    }

    public DisambiguationJob_Toy setFEATURES_DESCRIPTION(String fEATURES_DESCRIPTION) {
        FEATURES_DESCRIPTION = fEATURES_DESCRIPTION;
        return this;
    }

    public DisambiguationJob_Toy setREDUCER_NUM(int rEDUCER_NUM) {
        REDUCER_NUM = rEDUCER_NUM;
        return this;
    }

    public DisambiguationJob_Toy setTHRESHOLD(String tHRESHOLD) {
        THRESHOLD = tHRESHOLD;
        return this;
    }

    @Override
    public void setConf(Configuration conf) {
        this.conf = conf;
    }

    @Override
    public Configuration getConf() {
        return conf;
    }

    private void parseArgs(String[] args) {
        if (args == null || args.length != 5) {
            logger.debug("# of parameters is not equal to 5");
            logger.debug("You need to provide:");
            logger.debug("* an input table name");
            logger.debug("* an output table name");
            logger.debug("* a feature description string, e.g. FeatureXName#FeatureXExtractorName#FeatureXWeight#FeaturXMaxValue[,FeatureYName#FeatureYExtractorName#FeatureYWeight#FeaturYMaxValue]    ");
            logger.debug("* a disambiguation threshold");
            logger.debug("* a job name");
            logger.debug("");
            logger.debug("Default values will be used:");
            logger.debug("* grotoap10");
            logger.debug("* grotoap10-out");
            logger.debug("* EmailFeature#DocumentProto2EmailExtractor#0.81#1,"
                    + "KeywordFeature#DocumentProto2KeyWordExtractor#0.13#33");
            logger.debug("* -0.846161134713438d");
            logger.debug("* DisambiguationJob_Toy");

            args = new String[5];
            args[0] = "grotoap10";
            args[1] = "grotoap10-out2";
            args[2] = "EmailDisambiguator#DocumentProto2EmailExtractor#0.81#1,"
                    + "KeywordDisambiguator#DocumentProto2KeyWordExtractor#0.13#33";
            args[3] = "-0.846161134713438d";
            args[4] = "DisambiguationJob_Toy";
        }

        setINPUT_TABLE(args[0]);
        setOUTPUT_TABLE(args[1]);
        setFEATURES_DESCRIPTION(args[2]);
        setTHRESHOLD(args[3]);
        setNAME(args[4]);
    }

    /*
     * 
     * Job configuration and ignition
     * 
     */
    @Override
    public int run(String[] args) throws Exception {

        String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
        parseArgs(otherArgs);
        
        /*
         * First job configuration
         */
        conf.set("FEATURE_DESCRIPTION", FEATURES_DESCRIPTION);
        conf.set("THRESHOLD", THRESHOLD);
        //setting input table
        conf.set(TableInputFormat.INPUT_TABLE, INPUT_TABLE);

        //job creation
        Job job = new Job(conf);
        job.setJobName(NAME + " (input: " + INPUT_TABLE + ")");
        job.setJarByClass(DisambiguationJob_Toy.class);
        //scan for relevant data
        Scan scan = new Scan();<commit_msg>corrected usage of generic option parser<commit_after>/*
 * (C) 2010-2012 ICM UW. All rights reserved.
 */
package pl.edu.icm.coansys.disambiguation.author.jobs.hdfs;

import java.lang.management.ManagementFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.mapreduce.TableInputFormat;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.log4j.Logger;
import pl.edu.icm.coansys.disambiguation.auxil.LoggingInDisambiguation;
import pl.edu.icm.coansys.disambiguation.auxil.TextTextArrayMapWritable;
import pl.edu.icm.coansys.importers.constants.HBaseConstant;

/**
 *
 * @author pdendek
 * @version 1.0
 * @since 2012-08-07
 */
public class DisambiguationJob_Toy extends Configured implements Tool {

    /*
     * 
     * Inner fields
     * 
     */
    private static Logger logger = Logger.getLogger(LoggingInDisambiguation.class);
    private Configuration conf;
    /*
     * 
     * Fields to be set
     * 
     */
    String INPUT_TABLE = null;
    String OUTPUT_DIR = null;
    String FEATURES_DESCRIPTION = null;
    String NAME = null;
    String THRESHOLD = null;
    int REDUCER_NUM = 65;

    /*
     * 
     * Getters and setters
     * 
     */
    public String getINPUT_TABLE() {
        return INPUT_TABLE;
    }

    public DisambiguationJob_Toy setINPUT_TABLE(String iNPUT_TABLE) {
        INPUT_TABLE = iNPUT_TABLE;
        return this;
    }

    public String getOUTPUT_DIR() {
        return OUTPUT_DIR;
    }

    public DisambiguationJob_Toy setOUTPUT_TABLE(String oUTPUT_TABLE) {
        OUTPUT_DIR = oUTPUT_TABLE;
        return this;
    }

    public String getNAME() {
        return NAME;
    }

    public DisambiguationJob_Toy setNAME(String nAME) {
        NAME = nAME;
        return this;
    }

    public String getFEATURES_DESCRIPTION() {
        return FEATURES_DESCRIPTION;
    }

    public DisambiguationJob_Toy setFEATURES_DESCRIPTION(String fEATURES_DESCRIPTION) {
        FEATURES_DESCRIPTION = fEATURES_DESCRIPTION;
        return this;
    }

    public DisambiguationJob_Toy setREDUCER_NUM(int rEDUCER_NUM) {
        REDUCER_NUM = rEDUCER_NUM;
        return this;
    }

    public DisambiguationJob_Toy setTHRESHOLD(String tHRESHOLD) {
        THRESHOLD = tHRESHOLD;
        return this;
    }

    @Override
    public void setConf(Configuration conf) {
        this.conf = conf;
    }

    @Override
    public Configuration getConf() {
        return conf;
    }

    private void parseArgs(String[] args) {
        if (args == null || args.length != 5) {
            logger.debug("# of parameters is not equal to 5");
            logger.debug("You need to provide:");
            logger.debug("* an input table name");
            logger.debug("* an output table name");
            logger.debug("* a feature description string, e.g. FeatureXName#FeatureXExtractorName#FeatureXWeight#FeaturXMaxValue[,FeatureYName#FeatureYExtractorName#FeatureYWeight#FeaturYMaxValue]    ");
            logger.debug("* a disambiguation threshold");
            logger.debug("* a job name");
            logger.debug("");
            logger.debug("Default values will be used:");
            logger.debug("* grotoap10");
            logger.debug("* grotoap10-out");
            logger.debug("* EmailFeature#DocumentProto2EmailExtractor#0.81#1,"
                    + "KeywordFeature#DocumentProto2KeyWordExtractor#0.13#33");
            logger.debug("* -0.846161134713438d");
            logger.debug("* DisambiguationJob_Toy");

            args = new String[5];
            args[0] = "grotoap10";
            args[1] = "grotoap10-out2";
            args[2] = "EmailDisambiguator#DocumentProto2EmailExtractor#0.81#1,"
                    + "KeywordDisambiguator#DocumentProto2KeyWordExtractor#0.13#33";
            args[3] = "-0.846161134713438d";
            args[4] = "DisambiguationJob_Toy";
        }

        setINPUT_TABLE(args[0]);
        setOUTPUT_TABLE(args[1]);
        setFEATURES_DESCRIPTION(args[2]);
        setTHRESHOLD(args[3]);
        setNAME(args[4]);
    }

    /*
     * 
     * Job configuration and ignition
     * 
     */
    @Override
    public int run(String[] args) throws Exception {
        parseArgs(args);
        
        /*
         * First job configuration
         */
        conf.set("FEATURE_DESCRIPTION", FEATURES_DESCRIPTION);
        conf.set("THRESHOLD", THRESHOLD);
        //setting input table
        conf.set(TableInputFormat.INPUT_TABLE, INPUT_TABLE);

        //job creation
        Job job = new Job(conf);
        job.setJobName(NAME + " (input: " + INPUT_TABLE + ")");
        job.setJarByClass(DisambiguationJob_Toy.class);
        //scan for relevant data
        Scan scan = new Scan();
{'fraction_non_alphanumeric': 0.10502583372475341, 'fraction_numerical': 0.018224518553311413, 'mean_word_length': 2.6483893077450307, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4746181', 'n_tokens_mistral': 3431, 'n_tokens_neox': 3225, 'n_words': 615}