<commit_before>package com.scienceminer.nerd.disambiguation;

import com.scienceminer.nerd.exceptions.NerdException;
import com.scienceminer.nerd.utilities.NerdProperties;
import com.scienceminer.nerd.service.NerdQuery;
import com.scienceminer.nerd.utilities.StringPos;
import com.scienceminer.nerd.utilities.TextUtilities;
import com.scienceminer.nerd.utilities.Utilities;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory; 

import java.io.*;
import java.util.*;
import java.util.regex.*;

import com.googlecode.clearnlp.component.AbstractComponent;
import com.googlecode.clearnlp.segmentation.AbstractSegmenter;
import com.googlecode.clearnlp.engine.EngineGetter;
import com.googlecode.clearnlp.reader.AbstractReader;
import com.googlecode.clearnlp.tokenization.AbstractTokenizer;

import org.grobid.core.analyzers.GrobidAnalyzer;
import org.grobid.core.utilities.OffsetPosition;
import org.grobid.core.data.Entity;
import org.grobid.core.data.Sense;
import org.grobid.core.engines.NERParsers;
import org.grobid.core.exceptions.GrobidException;
import org.grobid.core.factory.*;
import org.grobid.core.mock.*;
import org.grobid.core.main.*;
import org.grobid.core.utilities.GrobidProperties;
import org.grobid.core.layout.LayoutToken;
import org.grobid.core.utilities.LayoutTokensUtil;
import org.grobid.core.utilities.BoundingBoxCalculator;
import org.grobid.core.lang.Language;
import org.grobid.core.layout.BoundingBox;
import org.grobid.core.utilities.LanguageUtilities;

import com.scienceminer.nerd.utilities.Stopwords;

import static org.apache.commons.lang3.StringUtils.isBlank;

/**
 * 
 * Everything we need to get the mentions and names entities from a text. From a text or a 
 * NerdQuery object, we generate a list of Entity objects corresponding to the potential 
 * mentions of entity to be considred by the further disambiguation stage. 
 */
public class ProcessText {
	public final String language = AbstractReader.LANG_EN;
	
	private static final Logger LOGGER = LoggerFactory
			.getLogger(ProcessText.class);
	
  	private static volatile ProcessText instance;
	
	// ClearParser components for sentence segmentation
	private AbstractTokenizer tokenizer = null;
	
	// GROBID tokenizer
	//private GrobidAnalyzer analyzer = GrobidAnalyzer.getInstance(); 

	private NERParsers nerParsers = null;

	private Stopwords stopwords = Stopwords.getInstance();
	
	public static final int NGRAM_LENGTH = 6;

	// default indo-european delimiters, should be moved to language specific analysers
	private static String delimiters = " \n\t" + TextUtilities.fullPunctuations;

	public static ProcessText getInstance() throws Exception {<commit_msg>Solve conflicts + merge<commit_after>package com.scienceminer.nerd.disambiguation;

import com.scienceminer.nerd.exceptions.NerdException;
import com.scienceminer.nerd.utilities.NerdProperties;
import com.scienceminer.nerd.service.NerdQuery;
import com.scienceminer.nerd.utilities.StringPos;
import com.scienceminer.nerd.utilities.TextUtilities;
import com.scienceminer.nerd.utilities.Utilities;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory; 

import java.io.*;
import java.util.*;
import java.util.regex.*;

import com.googlecode.clearnlp.component.AbstractComponent;
import com.googlecode.clearnlp.segmentation.AbstractSegmenter;
import com.googlecode.clearnlp.engine.EngineGetter;
import com.googlecode.clearnlp.reader.AbstractReader;
import com.googlecode.clearnlp.tokenization.AbstractTokenizer;

import org.grobid.core.analyzers.GrobidAnalyzer;
import org.grobid.core.utilities.OffsetPosition;
import org.grobid.core.data.Entity;
import org.grobid.core.data.Sense;
import org.grobid.core.engines.NERParsers;
import org.grobid.core.exceptions.GrobidException;
import org.grobid.core.factory.*;
import org.grobid.core.mock.*;
import org.grobid.core.main.*;
import org.grobid.core.utilities.GrobidProperties;
import org.grobid.core.layout.LayoutToken;
import org.grobid.core.utilities.LayoutTokensUtil;
import org.grobid.core.utilities.BoundingBoxCalculator;
import org.grobid.core.lang.Language;
import org.grobid.core.layout.BoundingBox;
import org.grobid.core.utilities.LanguageUtilities;

import com.scienceminer.nerd.utilities.Stopwords;

import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.apache.commons.lang3.StringUtils.isNotBlank;

/**
 * 
 * Everything we need to get the mentions and names entities from a text. From a text or a 
 * NerdQuery object, we generate a list of Entity objects corresponding to the potential 
 * mentions of entity to be considred by the further disambiguation stage. 
 */
public class ProcessText {
	public final String language = AbstractReader.LANG_EN;
	
	private static final Logger LOGGER = LoggerFactory
			.getLogger(ProcessText.class);
	
  	private static volatile ProcessText instance;
	
	// ClearParser components for sentence segmentation
	private AbstractTokenizer tokenizer = null;
	
	// GROBID tokenizer
	//private GrobidAnalyzer analyzer = GrobidAnalyzer.getInstance(); 

	private NERParsers nerParsers = null;

	private Stopwords stopwords = Stopwords.getInstance();
	
	public static final int NGRAM_LENGTH = 6;

	// default indo-european delimiters, should be moved to language specific analysers
	private static String delimiters = " \n\t" + TextUtilities.fullPunctuations;

	public static ProcessText getInstance() throws Exception {
{'fraction_non_alphanumeric': 0.08924485125858124, 'fraction_numerical': 0.002288329519450801, 'mean_word_length': 5.576388888888889, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3523284', 'n_tokens_mistral': 1745, 'n_tokens_neox': 1687, 'n_words': 385}