<commit_before>package de.uni_potsdam.hpi.coheel.ml

import de.uni_potsdam.hpi.coheel.programs.DataClasses.{ClassificationInfo, FeatureLine}
import weka.classifiers.Classifier
import weka.core.{Instances, Attribute, FastVector, Instance}
import scala.collection.mutable

object CoheelClassifier {
	val POS_TAG_GROUPS = Array(
		List("NN", "NNS"),
		List("NNP", "NNPS"),
		List("JJ", "JJR", "JJS"),
		List("VB", "VBD", "VBG", "VBN", "VBP", "VBZ"),
		List("CD"),
		List("SYM"),
		List("WDT", "WP", "WP$", "WRB")
	)

	val FEATURE_DEFINITION = {
		val attrs = new FastVector(17)
		attrs.addElement(new Attribute("id"))

		attrs.addElement(new Attribute("NN"))
		attrs.addElement(new Attribute("NNP"))
		attrs.addElement(new Attribute("JJ"))
		attrs.addElement(new Attribute("VB"))
		attrs.addElement(new Attribute("CD"))
		attrs.addElement(new Attribute("SYM"))
		attrs.addElement(new Attribute("W"))

		attrs.addElement(new Attribute("prom"))
		attrs.addElement(new Attribute("promRank"))
		attrs.addElement(new Attribute("promDeltaTop"))
		attrs.addElement(new Attribute("promDeltaSucc"))
		attrs.addElement(new Attribute("context"))
		attrs.addElement(new Attribute("contextRank"))
		attrs.addElement(new Attribute("contextDeltaTop"))
		attrs.addElement(new Attribute("contextDeltaSucc"))
		val classAttrValues = new FastVector(2)
		classAttrValues.addElement("0.0")
		classAttrValues.addElement("1.0")
		val classAttr = new Attribute("class", classAttrValues)
		attrs.addElement(classAttr)
		attrs
	}
	val FEATURE_DEFINITION_WITHOUT_ID = {
		val newDefinition = FEATURE_DEFINITION.copy().asInstanceOf[FastVector]
		// remove id
		newDefinition.removeElementAt(0)
//		// remove class attribute
//		newDefinition.removeElementAt(newDefinition.size() - 1)
		newDefinition
	}
}

class CoheelClassifier(classifier: Classifier) {

	val NUMBER_OF_FEATURES = 15
	val POSITIVE_CLASS = 1.0

	val instances = new Instances("Classification", CoheelClassifier.FEATURE_DEFINITION_WITHOUT_ID, 1)
	instances.setClassIndex(NUMBER_OF_FEATURES)

	/**
	 * Classifies a given group of instances, which result from the same link/trie hit in the original text
	 * @param featureLine The features of all possible links.
	 * @return The predicted link or None, if no link is predicted.
	 */
	def classifyResults(featureLine: mutable.ArrayBuffer[FeatureLine[ClassificationInfo]]): Option[FeatureLine[ClassificationInfo]] = {
		var positivePredictions = List[FeatureLine[ClassificationInfo]]()
		featureLine.foreach { featureLine =>
			assert(featureLine.features.size == NUMBER_OF_FEATURES)
			val instance = buildInstance(featureLine)
			instance.setDataset(instances)
			if (classifier.classifyInstance(instance) == POSITIVE_CLASS) {
				positivePredictions ::= featureLine
			}
		}
		// TODO: Change to return only if there is _exactly_ one positive prediction
		if (positivePredictions.size >= 1)
			positivePredictions.headOption
		else
			None
	}<commit_msg>Allow more than one seed per trie hit for now.<commit_after>package de.uni_potsdam.hpi.coheel.ml

import de.uni_potsdam.hpi.coheel.programs.DataClasses.{ClassificationInfo, FeatureLine}
import weka.classifiers.Classifier
import weka.core.{Instances, Attribute, FastVector, Instance}
import scala.collection.mutable

object CoheelClassifier {
	val POS_TAG_GROUPS = Array(
		List("NN", "NNS"),
		List("NNP", "NNPS"),
		List("JJ", "JJR", "JJS"),
		List("VB", "VBD", "VBG", "VBN", "VBP", "VBZ"),
		List("CD"),
		List("SYM"),
		List("WDT", "WP", "WP$", "WRB")
	)

	val FEATURE_DEFINITION = {
		val attrs = new FastVector(17)
		attrs.addElement(new Attribute("id"))

		attrs.addElement(new Attribute("NN"))
		attrs.addElement(new Attribute("NNP"))
		attrs.addElement(new Attribute("JJ"))
		attrs.addElement(new Attribute("VB"))
		attrs.addElement(new Attribute("CD"))
		attrs.addElement(new Attribute("SYM"))
		attrs.addElement(new Attribute("W"))

		attrs.addElement(new Attribute("prom"))
		attrs.addElement(new Attribute("promRank"))
		attrs.addElement(new Attribute("promDeltaTop"))
		attrs.addElement(new Attribute("promDeltaSucc"))
		attrs.addElement(new Attribute("context"))
		attrs.addElement(new Attribute("contextRank"))
		attrs.addElement(new Attribute("contextDeltaTop"))
		attrs.addElement(new Attribute("contextDeltaSucc"))
		val classAttrValues = new FastVector(2)
		classAttrValues.addElement("0.0")
		classAttrValues.addElement("1.0")
		val classAttr = new Attribute("class", classAttrValues)
		attrs.addElement(classAttr)
		attrs
	}
	val FEATURE_DEFINITION_WITHOUT_ID = {
		val newDefinition = FEATURE_DEFINITION.copy().asInstanceOf[FastVector]
		// remove id
		newDefinition.removeElementAt(0)
//		// remove class attribute
//		newDefinition.removeElementAt(newDefinition.size() - 1)
		newDefinition
	}
}

class CoheelClassifier(classifier: Classifier) {

	val NUMBER_OF_FEATURES = 15
	val POSITIVE_CLASS = 1.0

	val instances = new Instances("Classification", CoheelClassifier.FEATURE_DEFINITION_WITHOUT_ID, 1)
	instances.setClassIndex(NUMBER_OF_FEATURES)

	/**
	 * Classifies a given group of instances, which result from the same link/trie hit in the original text
	 * @param featureLine The features of all possible links.
	 * @return The predicted link or None, if no link is predicted.
	 */
	def classifyResults(featureLine: mutable.ArrayBuffer[FeatureLine[ClassificationInfo]]): List[FeatureLine[ClassificationInfo]] = {
		var positivePredictions = List[FeatureLine[ClassificationInfo]]()
		featureLine.foreach { featureLine =>
			assert(featureLine.features.size == NUMBER_OF_FEATURES)
			val instance = buildInstance(featureLine)
			instance.setDataset(instances)
			if (classifier.classifyInstance(instance) == POSITIVE_CLASS) {
				positivePredictions ::= featureLine
			}
		}
//		// TODO: Change to return only if there is _exactly_ one positive prediction
//		if (positivePredictions.size >= 1)
//			positivePredictions.headOption
//		else
//			None
		// TODO: Currently returning all seeds!
		positivePredictions
	}
{'fraction_non_alphanumeric': 0.12522989466644374, 'fraction_numerical': 0.005015883631499749, 'mean_word_length': 5.53056768558952, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4670192', 'n_tokens_mistral': 2064, 'n_tokens_neox': 1900, 'n_words': 401}