<commit_before>/*
 * Copyright 2009 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *	 http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package relex.parser;

import java.util.ArrayList;
import java.util.HashMap;

import org.linkgrammar.LGService;
import org.linkgrammar.LinkGrammar;

import relex.ParsedSentence;
import relex.Sentence;
import relex.feature.FeatureNode;
import relex.feature.LinkView;
import relex.feature.LinkableView;
import relex.stats.SimpleTruthValue;

public class LocalLGParser extends LGParser
{
	private static final int verbosity = 0;
	private static final double min_score = -0.001;
	private static final double score_bump = 0.001;

	private ThreadLocal<Boolean> initialized = new ThreadLocal<Boolean>()
	{
		protected Boolean initialValue()
		{
			return Boolean.FALSE;
		}
	};

	public void init()
	{
		// Must set language or dictionary path BEFORE initializing!
		if (_lang != null)
			LinkGrammar.setLanguage(_lang);

		if (_dict_path != null)
			LinkGrammar.setDictionariesPath(_dict_path);

		if (!initialized.get())
			LinkGrammar.init();

		LGService.configure(_config);
		initialized.set(Boolean.TRUE);
	}

	public void close()
	{
		LinkGrammar.close();
		initialized.set(Boolean.FALSE);
	}

	public Sentence parse(String sentence) throws ParseException
	{
		if (!initialized.get())
			init();
		Long starttime;
		if (verbosity > 0) starttime = System.currentTimeMillis();

		Sentence sntc = new Sentence();

		boolean ignoreFirst = false; // true if first word is LEFT-WALL
		boolean ignoreLast = false;  // true if first word is RIGHT_WALL
		if (verbosity >= 5) System.err.println("about to parse [" + sentence + "]");
		LinkGrammar.parse(sentence);
		if (verbosity >= 5) System.err.println("parsed [" + sentence + "]");

		int numParses = LinkGrammar.getNumLinkages();
		if (verbosity >= 5) System.err.println("found " + numParses + " parse(s)");

		ArrayList<ParsedSentence> parses = new ArrayList<ParsedSentence>();

		if ((numParses < 1) ||
		    (!_config.isAllowSkippedWords() && LinkGrammar.getNumSkippedWords() > 0))
		{
			System.err.println("Warning: No parses found for:\n" +
			     sentence);
			return sntc;
		}

		for (int i = 0; i < numParses && i < _config.getMaxLinkages(); i++)
		{
			if (verbosity >= 5) System.err.println("making linkage for parse " + i);
			LinkGrammar.makeLinkage(i);

			if (verbosity >= 5) System.err.println("making sentence for parse " + i);
			ParsedSentence s = new ParsedSentence(sentence);

			// Add words
			int numWords = LinkGrammar.getNumWords();
			FeatureNode lastFN = null;
			FeatureNode leftWall = null;
			/*
			 * Note that we should adjust cNumWords to count the words in the
			 * sentence -- this will have other consequences when we try to do
			 * partial parses -- must take this action carefully
			 */

			/*
			 * In order to find the start character of each word/token, we need
			 * to keep track of previous instances of the word (in case the same
			 * word appears more than once in the sentence). The variables
			 * "index" and "timesTokenSeen" are used for this:
			 */
			int startChar = 0;
			HashMap<String,Integer> timesTokenSeen = new HashMap<String,Integer>();
			// System.err.println("LINKPARSER NUMWORDS=" + numWords + "
			// SKIPPED=" + cNumSkippedWords());
			for (int w = 0; w < numWords; w++)
			{
				String wordString = LinkGrammar.getLinkageWord(w);
				if (verbosity >= 5) System.err.println(" Processing Word " + wordString);

				if (wordString.equals("RIGHT-WALL"))
				{
					ignoreLast = true;
				}
				else
				{
					FeatureNode fn = new FeatureNode();
					if (wordString.equals("LEFT-WALL")) leftWall = fn;
					// LEFT-WALL should always be first word, so throw an
					// exception if it was not.
					if (leftWall == null)
						throw new RuntimeException("Invalid parse: " +
							"first word is not left wall");

					// Set the word and part-of-speech
					LinkableView.setWordAndPos(fn, wordString);

					// Create a feature "this" which points to the linkable
					fn.set("this", fn);<commit_msg>fix for russian<commit_after>/*
 * Copyright 2009 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *	 http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package relex.parser;

import java.util.ArrayList;
import java.util.HashMap;

import org.linkgrammar.LGService;
import org.linkgrammar.LinkGrammar;

import relex.ParsedSentence;
import relex.Sentence;
import relex.feature.FeatureNode;
import relex.feature.LinkView;
import relex.feature.LinkableView;
import relex.stats.SimpleTruthValue;

public class LocalLGParser extends LGParser
{
	private static final int verbosity = 0;
	private static final double min_score = -0.001;
	private static final double score_bump = 0.001;

	private ThreadLocal<Boolean> initialized = new ThreadLocal<Boolean>()
	{
		protected Boolean initialValue()
		{
			return Boolean.FALSE;
		}
	};

	public void init()
	{
		// Must set language or dictionary path BEFORE initializing!
		if (_lang != null)
			LinkGrammar.setLanguage(_lang);

		if (_dict_path != null)
			LinkGrammar.setDictionariesPath(_dict_path);

		if (!initialized.get())
			LinkGrammar.init();

		LGService.configure(_config);
		initialized.set(Boolean.TRUE);
	}

	public void close()
	{
		LinkGrammar.close();
		initialized.set(Boolean.FALSE);
	}

	public Sentence parse(String sentence) throws ParseException
	{
		if (!initialized.get())
			init();
		Long starttime;
		if (verbosity > 0) starttime = System.currentTimeMillis();

		Sentence sntc = new Sentence();

		boolean ignoreFirst = false; // true if first word is LEFT-WALL
		boolean ignoreLast = false;  // true if first word is RIGHT_WALL
		if (verbosity >= 5) System.err.println("about to parse [" + sentence + "]");
		LinkGrammar.parse(sentence);
		if (verbosity >= 5) System.err.println("parsed [" + sentence + "]");

		int numParses = LinkGrammar.getNumLinkages();
		if (verbosity >= 5) System.err.println("found " + numParses + " parse(s)");

		ArrayList<ParsedSentence> parses = new ArrayList<ParsedSentence>();

		if ((numParses < 1) ||
		    (!_config.isAllowSkippedWords() && LinkGrammar.getNumSkippedWords() > 0))
		{
			System.err.println("Warning: No parses found for:\n" +
			     sentence);
			return sntc;
		}

		for (int i = 0; i < numParses && i < _config.getMaxLinkages(); i++)
		{
			if (verbosity >= 5) System.err.println("making linkage for parse " + i);
			LinkGrammar.makeLinkage(i);

			if (verbosity >= 5) System.err.println("making sentence for parse " + i);
			ParsedSentence s = new ParsedSentence(sentence);

			// Add words
			int numWords = LinkGrammar.getNumWords();
			FeatureNode lastFN = null;
			FeatureNode leftWall = null;
			/*
			 * Note that we should adjust cNumWords to count the words in the
			 * sentence -- this will have other consequences when we try to do
			 * partial parses -- must take this action carefully
			 */

			/*
			 * In order to find the start character of each word/token, we need
			 * to keep track of previous instances of the word (in case the same
			 * word appears more than once in the sentence). The variables
			 * "index" and "timesTokenSeen" are used for this:
			 */
			int startChar = 0;
			HashMap<String,Integer> timesTokenSeen = new HashMap<String,Integer>();
			// System.err.println("LINKPARSER NUMWORDS=" + numWords + "
			// SKIPPED=" + cNumSkippedWords());
			for (int w = 0; w < numWords; w++)
			{
				String wordString = LinkGrammar.getLinkageWord(w);
				if (verbosity >= 5) System.err.println(" Processing Word " + wordString);

				// In Russian, some words (suffixes) can be zero-length.
				if (0 == wordString.length()) continue;

				if (wordString.equals("RIGHT-WALL"))
				{
					ignoreLast = true;
				}
				else
				{
					FeatureNode fn = new FeatureNode();
					if (wordString.equals("LEFT-WALL")) leftWall = fn;
					// LEFT-WALL should always be first word, so throw an
					// exception if it was not.
					if (leftWall == null)
						throw new RuntimeException("Invalid parse: " +
							"first word is not left wall");

					// Set the word and part-of-speech
					LinkableView.setWordAndPos(fn, wordString);

					// Create a feature "this" which points to the linkable
					fn.set("this", fn);
{'fraction_non_alphanumeric': 0.1118766404199475, 'fraction_numerical': 0.0064523184601924756, 'mean_word_length': 3.5160493827160493, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4810892', 'n_tokens_mistral': 3219, 'n_tokens_neox': 2747, 'n_words': 901}