<commit_before>package edu.isi.karma.semantictypes.typinghandler;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.lucene.queryparser.classic.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import edu.isi.karma.modeling.semantictypes.ISemanticTypeModelHandler;
import edu.isi.karma.modeling.semantictypes.SemanticTypeLabel;
import edu.isi.karma.semantictypes.tfIdf.Indexer;
import edu.isi.karma.semantictypes.tfIdf.Searcher;
import edu.isi.karma.webserver.ServletContextParameterMap;
import edu.isi.karma.webserver.ServletContextParameterMap.ContextParameter;

/**
 * This is the API class for the semantic typing module, implementing the
 * combined approach of TF-IDF based cosine similarity and Kolmogorov-Smirnov
 * test approaches for textual and numeric respectively by Ramnandan.S.K and
 * <NAME>.
 * 
 * @author ramnandan
 * 
 */

public class LuceneBasedSTModelHandler implements ISemanticTypeModelHandler {

	/**
	 * This internal class represents an example.
	 */
	static class Example {
		String exampleString;

		/**
		 * @param exampleString
		 *            The string that the example represents No ColumnFeatures
		 *            specified.
		 */
		public Example(String exampleString) {
			this.exampleString = exampleString;
		}

		public String getString() {
			return exampleString;
		}

	}

	// ****************************************************************************************//

	// instance variables
	static Logger logger = LoggerFactory
			.getLogger(LuceneBasedSTModelHandler.class.getSimpleName());
	private ArrayList<String> allowedCharacters;

	private boolean modelEnabled = false;
	private String indexDirectory;

	/**
	 * NOTE: Currently, TF-IDF based approach is used for both textual and
	 * numeric data due to bug in KS test on Apache Commons Math.
	 * 
	 * TODO: Integrate KS test when this bug is resolved :
	 * https://issues.apache.org/jira/browse/MATH-1131
	 */

	public LuceneBasedSTModelHandler() {
		allowedCharacters = allowedCharacters();
		indexDirectory = ServletContextParameterMap
				.getParameterValue(ContextParameter.SEMTYPE_MODEL_DIRECTORY);
	}

	/**
	 * Adds the passed list of examples for training
	 * 
	 * @param label
	 *            True label for the list of example.
	 * @param examples
	 *            List of example strings.
	 * @return True if success, else False
	 */
	@Override
	public synchronized boolean addType(String label, List<String> examples) {
		ArrayList<String> cleanedExamples;
		ArrayList<Example> selectedExamples;
		boolean savingSuccessful = false;

		// running basic sanity checks in the input arguments
		if (label == null || label.trim().length() == 0 || examples.size() == 0) {
			logger.warn("@label argument cannot be null or an empty string and the @examples list cannot be empty.");
			return false;
		}
		label = label.trim();
		cleanedExamples = new ArrayList<String>();
		cleanedExamplesList(examples, cleanedExamples);
		examples = cleanedExamples;
		// making sure that the condition where the examples list is not empty
		// but contains junk only is not accepted
		if (examples.size() == 0) {
			logger.warn("@examples list contains forbidden characters only. The allowed characters are "
					+ allowedCharacters);
			return false;
		}

		// adding all the new examples to list of existing examples for the arg
		// label.
		selectedExamples = new ArrayList<Example>();
		for (String newExampleString : examples) {
			Example newExample = new Example(newExampleString);
			selectedExamples.add(newExample);
		}

		// if the column is textual
		try {
			savingSuccessful = indexTrainingColumn(label, selectedExamples);
		} catch (IOException e) {
			e.printStackTrace();
		}

		return savingSuccessful;
	}

	/**
	 * Indexes the given training column for a specific label
	 * 
	 * @param label
	 * @param selectedExamples
	 * @return
	 * @throws IOException
	 */
	private boolean indexTrainingColumn(String label,
			ArrayList<Example> selectedExamples) throws IOException {
		// add documents to index
		StringBuilder sb = new StringBuilder();
		for (Example ex : selectedExamples) {
			sb.append(ex.getString());
			sb.append(" ");
		}

		boolean labelExists = false;
		if (indexDirectoryExists()) {
			try {
				// check if semantic label already exists in index
				Searcher searcher = new Searcher(indexDirectory,
						Indexer.LABEL_FIELD_NAME);
				try {
					labelExists = searcher.existsSemanticLabel(label);
				} finally {
					searcher.close();
				}
			} catch (Exception e) {
				// Ignore, the searcher might not work if index is empty.<commit_msg>remove Example class.<commit_after>package edu.isi.karma.semantictypes.typinghandler;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.lucene.queryparser.classic.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import edu.isi.karma.modeling.semantictypes.ISemanticTypeModelHandler;
import edu.isi.karma.modeling.semantictypes.SemanticTypeLabel;
import edu.isi.karma.semantictypes.tfIdf.Indexer;
import edu.isi.karma.semantictypes.tfIdf.Searcher;
import edu.isi.karma.webserver.ServletContextParameterMap;
import edu.isi.karma.webserver.ServletContextParameterMap.ContextParameter;

/**
 * This is the API class for the semantic typing module, implementing the
 * combined approach of TF-IDF based cosine similarity and Kolmogorov-Smirnov
 * test approaches for textual and numeric respectively by Ramnandan.S.K and
 * <NAME>.
 * 
 * @author ramnandan
 * 
 */

public class LuceneBasedSTModelHandler implements ISemanticTypeModelHandler {

	static Logger logger = LoggerFactory
			.getLogger(LuceneBasedSTModelHandler.class.getSimpleName());
	private ArrayList<String> allowedCharacters;

	private boolean modelEnabled = false;
	private String indexDirectory;

	/**
	 * NOTE: Currently, TF-IDF based approach is used for both textual and
	 * numeric data due to bug in KS test on Apache Commons Math.
	 * 
	 * TODO: Integrate KS test when this bug is resolved :
	 * https://issues.apache.org/jira/browse/MATH-1131
	 */

	public LuceneBasedSTModelHandler() {
		allowedCharacters = allowedCharacters();
		indexDirectory = ServletContextParameterMap
				.getParameterValue(ContextParameter.SEMTYPE_MODEL_DIRECTORY);
	}

	/**
	 * Adds the passed list of examples for training
	 * 
	 * @param label
	 *            True label for the list of example.
	 * @param examples
	 *            List of example strings.
	 * @return True if success, else False
	 */
	@Override
	public synchronized boolean addType(String label, List<String> examples) {
		boolean savingSuccessful = false;

		// running basic sanity checks in the input arguments
		if (label == null || label.trim().length() == 0 || examples.size() == 0) {
			logger.warn("@label argument cannot be null or an empty string and the @examples list cannot be empty.");
			return false;
		}
		
		label = label.trim();
		ArrayList<String> cleanedExamples = new ArrayList<String>();
		cleanedExamplesList(examples, cleanedExamples);
		
		// making sure that the condition where the examples list is not empty
		// but contains junk only is not accepted
		if (cleanedExamples.size() == 0) {
			logger.warn("@examples list contains forbidden characters only. The allowed characters are "
					+ allowedCharacters);
			return false;
		}


		// if the column is textual
		try {
			savingSuccessful = indexTrainingColumn(label, cleanedExamples);
		} catch (IOException e) {
			e.printStackTrace();
		}

		return savingSuccessful;
	}

	/**
	 * Indexes the given training column for a specific label
	 * 
	 * @param label
	 * @param selectedExamples
	 * @return
	 * @throws IOException
	 */
	private boolean indexTrainingColumn(String label,
			ArrayList<String> selectedExamples) throws IOException {
		// add documents to index
		StringBuilder sb = new StringBuilder();
		for (String ex : selectedExamples) {
			sb.append(ex);
			sb.append(" ");
		}

		boolean labelExists = false;
		if (indexDirectoryExists()) {
			try {
				// check if semantic label already exists in index
				Searcher searcher = new Searcher(indexDirectory,
						Indexer.LABEL_FIELD_NAME);
				try {
					labelExists = searcher.existsSemanticLabel(label);
				} finally {
					searcher.close();
				}
			} catch (Exception e) {
				// Ignore, the searcher might not work if index is empty.
{'fraction_non_alphanumeric': 0.09997636492554951, 'fraction_numerical': 0.002127156700543607, 'mean_word_length': 3.8778097982708934, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3233931', 'n_tokens_mistral': 2679, 'n_tokens_neox': 2314, 'n_words': 726}