<commit_before>package cz.brmlab.yodaqa.provider.rdf;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;

import com.hp.hpl.jena.rdf.model.Literal;

import cz.brmlab.yodaqa.provider.JWordnet;

import net.didion.jwnl.data.IndexWord;
import net.didion.jwnl.data.Synset;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.slf4j.Logger;

/** A wrapper around DBpedia dataset that maps concepts to
 * dbpedia2:wordnet_type ontology pieces. This gives some evidence
 * of the type of the concept that we directly match with other
 * WordNet data.
 *
 * XXX: The WordNet types here also disambiguate word senses.
 * We drop this information for now, instead of converting this
 * to synset numbers. */

public class DBpediaWNTypes extends CachedJenaLookup {
	private static final Log logger =
		LogFactory.getLog(DBpediaWNTypes.class);

	/** Query for a given title, returning a set of types. The type is in
	 * the form of string/synset. */
	public List<String> query(String title, Logger logger) {
		for (String titleForm : cookedTitles(title)) {
			List<String> results = queryTitleForm(titleForm, logger);
			if (!results.isEmpty())
				return results;
		}
		return new ArrayList<String>();
	}

	/** Query for a given specific title form, returning a set
	 * of types. The type is in the form of string/synset. */
	public List<String> queryTitleForm(String title, Logger logger) {
		/* XXX: Case-insensitive search via SPARQL turns out
		 * to be surprisingly tricky.  Cover 90% of all cases
		 * by force-capitalizing the first letter in the sought
		 * after title. */
		title = Character.toUpperCase(title.charAt(0)) + title.substring(1);

		title = title.replaceAll("\"", "").replaceAll("\\\\", "").replaceAll("\n", " ");
		String rawQueryStr =
			"{\n" +
			   // (A) fetch resources with @title label
			"  ?res rdfs:label \"" + title + "\"@en.\n" +
			"} UNION {\n" +
			   // (B) fetch also resources targetted by @title redirect
			"  ?redir dbo:wikiPageRedirects ?res .\n" +
			"  ?redir rdfs:label \"" + title + "\"@en .\n" +
			"}\n" +
			 // set the output variable
			"?res dbpedia2:wordnet_type ?type .\n" +

			 // weed out resources that are categories and other in-namespace junk
			"FILTER ( !regex(str(?res), '^http://dbpedia.org/resource/[^_]*:', 'i') )\n" +
			"";
		//logger.debug("executing sparql query: {}", rawQueryStr);
		List<Literal[]> rawResults = rawQuery(rawQueryStr,
			new String[] { "type" });

		List<String> results = new LinkedList<String>();
		for (Literal[] rawResult : rawResults) {
			// XXX: we assume the type is always noun
			int senseIdx = Integer.parseInt(rawResult[0].getString().replaceAll("^.*-([^-]*)$", "$1"));
			String typeLabel = rawResult[0].getString().replaceAll("^synset-([^-]*)-.*$", "$1").replaceAll("_", " ");

			try {
				IndexWord w = JWordnet.getDictionary().lookupIndexWord(net.didion.jwnl.data.POS.NOUN, typeLabel);
				Synset s = w.getSenses()[senseIdx];
				long synset = s.getOffset();

				// logger.debug("DBpedia {} wntype: [[{}]]", title, typeLabel);
				results.add(typeLabel + "/" + Long.toString(synset));<commit_msg>DBpediaWNTypes: Fix senseIdx off-by-one bug (sense# is 1-based, not 0-based)<commit_after>package cz.brmlab.yodaqa.provider.rdf;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;

import com.hp.hpl.jena.rdf.model.Literal;

import cz.brmlab.yodaqa.provider.JWordnet;

import net.didion.jwnl.data.IndexWord;
import net.didion.jwnl.data.Synset;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.slf4j.Logger;

/** A wrapper around DBpedia dataset that maps concepts to
 * dbpedia2:wordnet_type ontology pieces. This gives some evidence
 * of the type of the concept that we directly match with other
 * WordNet data.
 *
 * XXX: The WordNet types here also disambiguate word senses.
 * We drop this information for now, instead of converting this
 * to synset numbers. */

public class DBpediaWNTypes extends CachedJenaLookup {
	private static final Log logger =
		LogFactory.getLog(DBpediaWNTypes.class);

	/** Query for a given title, returning a set of types. The type is in
	 * the form of string/synset. */
	public List<String> query(String title, Logger logger) {
		for (String titleForm : cookedTitles(title)) {
			List<String> results = queryTitleForm(titleForm, logger);
			if (!results.isEmpty())
				return results;
		}
		return new ArrayList<String>();
	}

	/** Query for a given specific title form, returning a set
	 * of types. The type is in the form of string/synset. */
	public List<String> queryTitleForm(String title, Logger logger) {
		/* XXX: Case-insensitive search via SPARQL turns out
		 * to be surprisingly tricky.  Cover 90% of all cases
		 * by force-capitalizing the first letter in the sought
		 * after title. */
		title = Character.toUpperCase(title.charAt(0)) + title.substring(1);

		title = title.replaceAll("\"", "").replaceAll("\\\\", "").replaceAll("\n", " ");
		String rawQueryStr =
			"{\n" +
			   // (A) fetch resources with @title label
			"  ?res rdfs:label \"" + title + "\"@en.\n" +
			"} UNION {\n" +
			   // (B) fetch also resources targetted by @title redirect
			"  ?redir dbo:wikiPageRedirects ?res .\n" +
			"  ?redir rdfs:label \"" + title + "\"@en .\n" +
			"}\n" +
			 // set the output variable
			"?res dbpedia2:wordnet_type ?type .\n" +

			 // weed out resources that are categories and other in-namespace junk
			"FILTER ( !regex(str(?res), '^http://dbpedia.org/resource/[^_]*:', 'i') )\n" +
			"";
		//logger.debug("executing sparql query: {}", rawQueryStr);
		List<Literal[]> rawResults = rawQuery(rawQueryStr,
			new String[] { "type" });

		List<String> results = new LinkedList<String>();
		for (Literal[] rawResult : rawResults) {
			// XXX: we assume the type is always noun
			int senseIdx = Integer.parseInt(rawResult[0].getString().replaceAll("^.*-([^-]*)$", "$1"));
			String typeLabel = rawResult[0].getString().replaceAll("^synset-([^-]*)-.*$", "$1").replaceAll("_", " ");

			try {
				IndexWord w = JWordnet.getDictionary().lookupIndexWord(net.didion.jwnl.data.POS.NOUN, typeLabel);
				Synset s = w.getSenses()[senseIdx - 1];
				long synset = s.getOffset();

				// logger.debug("DBpedia {} wntype: [[{}]]", title, typeLabel);
				results.add(typeLabel + "/" + Long.toString(synset));
{'fraction_non_alphanumeric': 0.1500686289461644, 'fraction_numerical': 0.003812719231355803, 'mean_word_length': 3.762527233115468, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '417064', 'n_tokens_mistral': 2324, 'n_tokens_neox': 2114, 'n_words': 601}