<commit_before>package graphene.hts.entityextraction;

import graphene.model.idl.G_Entity;
import graphene.model.idl.G_EntityTag;
import graphene.model.idl.G_Property;
import graphene.model.idl.G_PropertyTag;
import graphene.model.idl.G_Provenance;
import graphene.model.idl.G_Uncertainty;
import graphene.model.idlhelper.EntityHelper;
import graphene.model.idlhelper.PropertyHelper;
import graphene.util.validator.ValidationUtils;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public abstract class AbstractExtractor implements Extractor {

	private Matcher m;
	protected Pattern p;

	public Collection<String> extract(String source) {
		Set<String> matchList = new HashSet<String>();
		if (ValidationUtils.isValid(source)) {
			m = p.matcher(source);
			while (m.find()) {

				matchList.add(postProcessMatch(m.group(1)));

			}
		}
		return matchList;
	}

	/**
	 * Adds the abilty to post process the match so that it more closely
	 * resembles structured entities. Override in your concrete class if needed.
	 * 
	 * @param group
	 * @return
	 */
	public String postProcessMatch(String match) {
		return match;
	}

	@Override
	public Collection<G_Entity> extractEntities(String source) {
		Set<String> matchList = new HashSet<String>();
		if (ValidationUtils.isValid(source)) {
			m = p.matcher(source);
			while (m.find()) {
				matchList.add(postProcessMatch(m.group(1)));
			}
		}
		ArrayList<G_Entity> list = new ArrayList<G_Entity>();
		for (String id : matchList) {

			G_Provenance prov = new G_Provenance("Narrative");
			G_Uncertainty uncertainty = new G_Uncertainty();
			uncertainty.setConfidence(1.0d);
			G_Entity extractedIdentifierNode = new EntityHelper(id,
					getEntityTags(), prov, uncertainty, getProperties());
			list.add(postProcessEntity(extractedIdentifierNode));
		}
		return list;
	}

	/**
	 * Override in concrete classes to add additional properties to this entity.
	 * @param extractedIdentifierNode
	 * @return
	 */
	public G_Entity postProcessEntity(G_Entity extractedIdentifierNode) {
		return extractedIdentifierNode;
	}
}<commit_msg>Moved funnels<commit_after>package graphene.hts.entityextraction;

import graphene.model.idl.G_Entity;
import graphene.model.idl.G_Provenance;
import graphene.model.idl.G_Uncertainty;
import graphene.model.idlhelper.EntityHelper;
import graphene.util.validator.ValidationUtils;

import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * 
 * @author djue
 * 
 */
public abstract class AbstractExtractor implements Extractor {

	private Matcher m;
	protected Pattern p;

	public Collection<String> extract(String source) {
		Set<String> matchList = new HashSet<String>();
		if (ValidationUtils.isValid(source)) {
			m = p.matcher(source);
			while (m.find()) {

				matchList.add(postProcessMatch(m.group(1)));

			}
		}
		return matchList;
	}

	/**
	 * Adds the abilty to post process the match so that it more closely
	 * resembles structured entities. Override in your concrete class if needed.
	 * 
	 * @param group
	 * @return
	 */
	public String postProcessMatch(String match) {
		return match;
	}

	@Override
	public Collection<G_Entity> extractEntities(String source) {
		Set<String> matchList = new HashSet<String>();
		if (ValidationUtils.isValid(source)) {
			m = p.matcher(source);
			while (m.find()) {
				matchList.add(postProcessMatch(m.group(1)));
			}
		}
		ArrayList<G_Entity> list = new ArrayList<G_Entity>();
		for (String id : matchList) {

			G_Provenance prov = new G_Provenance("Narrative");
			G_Uncertainty uncertainty = new G_Uncertainty();
			uncertainty.setConfidence(1.0d);
			G_Entity extractedIdentifierNode = new EntityHelper(id,
					getEntityTags(), prov, uncertainty, getProperties());
			list.add(postProcessEntity(extractedIdentifierNode));
		}
		return list;
	}

	/**
	 * Override in concrete classes to add additional properties to this entity.
	 * 
	 * @param extractedIdentifierNode
	 * @return
	 */
	public G_Entity postProcessEntity(G_Entity extractedIdentifierNode) {
		return extractedIdentifierNode;
	}
}
{'fraction_non_alphanumeric': 0.10833333333333334, 'fraction_numerical': 0.001851851851851852, 'mean_word_length': 4.663171690694626, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4235360', 'n_tokens_mistral': 1485, 'n_tokens_neox': 1316, 'n_words': 313}