<commit_before>package lodVader.invalidLinks;

import java.util.ArrayList;
import java.util.HashMap;

import lodVader.mongodb.collections.gridFS.ObjectsBucket;
import lodVader.mongodb.collections.gridFS.SubjectsBucket;
import lodVader.mongodb.collections.gridFS.SuperBucket;

public class InvalidLinksFilters {

	HashMap<Integer, ArrayList<SuperBucket>> datasetSubjectFilters = new HashMap<Integer, ArrayList<SuperBucket>>();

	HashMap<Integer, ArrayList<SuperBucket>> datasetObjectFilters = new HashMap<Integer, ArrayList<SuperBucket>>();

	public void loadDatasetObjectFilter(int datasetID) {
		if(!datasetObjectFilters.containsKey(datasetID)){
			datasetObjectFilters.put(datasetID, new ObjectsBucket().getFiltersFromDataset(datasetID));
		}
	}
	public void loadDatasetSubjectFilter(int datasetID) {
		if(!datasetSubjectFilters.containsKey(datasetID)){
			datasetSubjectFilters.put(datasetID, new SubjectsBucket().getFiltersFromDataset(datasetID));
		}
	}

	public boolean queryDatasetSubject(String resource, int datasetID){
		for(SuperBucket buket: datasetSubjectFilters.get(datasetID)){
			if(buket.filter.compare(resource))
				return true;
		}
		return false;
	}<commit_msg>improved performance to load dataset filters<commit_after>package lodVader.invalidLinks;

import java.util.ArrayList;
import java.util.HashMap;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import lodVader.mongodb.collections.DatasetDB;
import lodVader.mongodb.collections.gridFS.ObjectsBucket;
import lodVader.mongodb.collections.gridFS.SubjectsBucket;
import lodVader.mongodb.collections.gridFS.SuperBucket;
import lodVader.threads.ProcessNSFromTuple;

public class InvalidLinksFilters {

	final static Logger logger = LoggerFactory.getLogger(InvalidLinksFilters.class);

	HashMap<Integer, ArrayList<SuperBucket>> datasetSubjectFilters = new HashMap<Integer, ArrayList<SuperBucket>>();

	HashMap<Integer, ArrayList<SuperBucket>> datasetObjectFilters = new HashMap<Integer, ArrayList<SuperBucket>>();

	public void loadDatasetObjectFilter(int datasetID) {
		if(!datasetObjectFilters.containsKey(datasetID)){
			datasetObjectFilters.put(datasetID, new ObjectsBucket().getFiltersFromDataset(datasetID));
		}
	}
	public void loadDatasetSubjectFilter(int datasetID) {
		if(!datasetSubjectFilters.containsKey(datasetID)){
			logger.info("Loading dataset"+new DatasetDB(datasetID).getTitle()+" to filter...");
			datasetSubjectFilters.put(datasetID, new SubjectsBucket().getFiltersFromDataset(datasetID));
		}
	}

	public boolean queryDatasetSubject(String resource, int datasetID){
		for(SuperBucket buket: datasetSubjectFilters.get(datasetID)){
			if(buket.filter.compare(resource))
				return true;
		}
		return false;
	}
{'fraction_non_alphanumeric': 0.10206337509211497, 'fraction_numerical': 0.0007369196757553427, 'mean_word_length': 7.564668769716088, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5627332', 'n_tokens_mistral': 810, 'n_tokens_neox': 789, 'n_words': 137}