<commit_before>/*******************************************************************************
 * Copyright (C) 2015 International Business Machines Corporation
 * All Rights Reserved
 *******************************************************************************/
package com.ibm.streamsx.sparkmllib;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Logger;

import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.mllib.linalg.Vector;
import org.apache.spark.mllib.linalg.Vectors;

import com.ibm.json.java.JSONObject;
import com.ibm.streams.operator.AbstractOperator;
import com.ibm.streams.operator.Attribute;
import com.ibm.streams.operator.OperatorContext;
import com.ibm.streams.operator.OperatorContext.ContextCheck;
import com.ibm.streams.operator.ProcessingElement;
import com.ibm.streams.operator.StreamSchema;
import com.ibm.streams.operator.StreamingInput;
import com.ibm.streams.operator.Tuple;
import com.ibm.streams.operator.Type;
import com.ibm.streams.operator.Type.MetaType;
import com.ibm.streams.operator.compile.OperatorContextChecker;
import com.ibm.streams.operator.logging.TraceLevel;
import com.ibm.streams.operator.logging.LoggerNames;
import com.ibm.streams.operator.meta.CollectionType;
import com.ibm.streams.operator.model.Libraries;
import com.ibm.streams.operator.model.Parameter;
import com.ibm.streams.operator.model.SharedLoader;

/**
 * Provides the base class for all Spark analytics operators. This base class is responsible
 * for the common parameters, common compiler checks, common utility methods as well
 * as setting up the spark context from the passed parameters so that the analytics models 
 * can be loaded.
 * 
 *  @param T This represents the specific Model class that is to be loaded by a specific 
 *  instance of this class.
 */
@SharedLoader
@Libraries({"@SPARK_HOME@/jars/*"})
public abstract class AbstractSparkMLlibOperator<T> extends AbstractOperator {

	
	private static final String CLASS_NAME =  AbstractSparkMLlibOperator.class.getName();
	/**
	 * Create a {@code Logger} specific to this class that will write to the SPL
	 * trace facility
	 */
	private static Logger tracer = Logger.getLogger(CLASS_NAME, "com.ibm.streamsx.sparkmllib.messages");

	private String modelPath;
	private String masterString;
	private JavaSparkContext javaContext;
	
	private T model;
	private Map<String, String> params;
	<commit_msg>remove SPARK_HOME also from annotation<commit_after>/*******************************************************************************
 * Copyright (C) 2015 International Business Machines Corporation
 * All Rights Reserved
 *******************************************************************************/
package com.ibm.streamsx.sparkmllib;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Logger;

import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.mllib.linalg.Vector;
import org.apache.spark.mllib.linalg.Vectors;

import com.ibm.json.java.JSONObject;
import com.ibm.streams.operator.AbstractOperator;
import com.ibm.streams.operator.Attribute;
import com.ibm.streams.operator.OperatorContext;
import com.ibm.streams.operator.OperatorContext.ContextCheck;
import com.ibm.streams.operator.ProcessingElement;
import com.ibm.streams.operator.StreamSchema;
import com.ibm.streams.operator.StreamingInput;
import com.ibm.streams.operator.Tuple;
import com.ibm.streams.operator.Type;
import com.ibm.streams.operator.Type.MetaType;
import com.ibm.streams.operator.compile.OperatorContextChecker;
import com.ibm.streams.operator.logging.TraceLevel;
import com.ibm.streams.operator.logging.LoggerNames;
import com.ibm.streams.operator.meta.CollectionType;
import com.ibm.streams.operator.model.Libraries;
import com.ibm.streams.operator.model.Parameter;
import com.ibm.streams.operator.model.SharedLoader;

/**
 * Provides the base class for all Spark analytics operators. This base class is responsible
 * for the common parameters, common compiler checks, common utility methods as well
 * as setting up the spark context from the passed parameters so that the analytics models 
 * can be loaded.
 * 
 *  @param T This represents the specific Model class that is to be loaded by a specific 
 *  instance of this class.
 */
@SharedLoader
@Libraries({"../../opt/downloaded/*"})
public abstract class AbstractSparkMLlibOperator<T> extends AbstractOperator {

	
	private static final String CLASS_NAME =  AbstractSparkMLlibOperator.class.getName();
	/**
	 * Create a {@code Logger} specific to this class that will write to the SPL
	 * trace facility
	 */
	private static Logger tracer = Logger.getLogger(CLASS_NAME, "com.ibm.streamsx.sparkmllib.messages");

	private String modelPath;
	private String masterString;
	private JavaSparkContext javaContext;
	
	private T model;
	private Map<String, String> params;
	
{'fraction_non_alphanumeric': 0.15011727912431588, 'fraction_numerical': 0.001563721657544957, 'mean_word_length': 5.636835278858626, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4446988', 'n_tokens_mistral': 1502, 'n_tokens_neox': 1436, 'n_words': 374}