<commit_before>/* This file is part of the Joshua Machine Translation System.
 * 
 * Joshua is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation; either version 2.1
 * of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free
 * Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 * MA 02111-1307 USA
 */
package edu.jhu.sa.util.suffix_array;

import joshua.util.ReverseOrder;
import joshua.util.sentence.Phrase;
import joshua.util.sentence.Vocabulary;

import java.util.*;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.io.*;


/**
 * SuffixArray is the main class for producing suffix arrays
 * from corpora, and manipulating them once created.  Suffix arrays
 * are a space economical way of storing a corpus and allowing
 * very quick searching of any substring within the corpus.  A
 * suffix array contains a list of references to every point in a
 * corpus, and each reference denotes the suffix starting at that
 * point and continuing to the end of the corpus.  The suffix array
 * is sorted alphabetically, so any substring within the corpus
 * can be found with a binary search in O(log n) time, where n is
 * the length of the corpus.
 *
 * @author  <NAME>
 * @since   10 December 2004
 * @author  <NAME>
 * @since   2 Jan 2005
 * @author  <NAME>
 * @since   9 February 2005
 * @version $LastChangedDate:2008-07-30 17:15:52 -0400 (Wed, 30 Jul 2008) $
 */
public class SuffixArray implements Corpus {

//===============================================================
// Constants
//===============================================================

	/**
	 * A random number generator used in the quick sort
	 * implementation.
	 */
	public static final Random RAND = new Random();
	
	
	/**
	 * The maximum length suffix to consider during sorting.
	 */
	public static int MAX_COMPARISON_LENGTH = 20;
	
	
	public static int INVERTED_INDEX_CAPACITY = 1000000;
	public static boolean POPULATE_INVERTED_INDEX = true;
	
	
	/** Logger for this class. */
	private static final Logger logger = 
		Logger.getLogger(SuffixArray.class.getName());
	
	
//===============================================================
// Member variables
//===============================================================

	protected int[] suffixes;
	protected CorpusArray corpus;
	
	protected InvertedIndex invertedIndex;
//===============================================================
// Constructor(s)
//===============================================================

	/** 
	 * Constructor takes a CorpusArray and creates a sorted
	 * suffix array from it.
	 */<commit_msg>Turned off precomputation of inverted index.<commit_after>/* This file is part of the Joshua Machine Translation System.
 * 
 * Joshua is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation; either version 2.1
 * of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free
 * Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 * MA 02111-1307 USA
 */
package edu.jhu.sa.util.suffix_array;

import joshua.util.ReverseOrder;
import joshua.util.sentence.Phrase;
import joshua.util.sentence.Vocabulary;

import java.util.*;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.io.*;


/**
 * SuffixArray is the main class for producing suffix arrays
 * from corpora, and manipulating them once created.  Suffix arrays
 * are a space economical way of storing a corpus and allowing
 * very quick searching of any substring within the corpus.  A
 * suffix array contains a list of references to every point in a
 * corpus, and each reference denotes the suffix starting at that
 * point and continuing to the end of the corpus.  The suffix array
 * is sorted alphabetically, so any substring within the corpus
 * can be found with a binary search in O(log n) time, where n is
 * the length of the corpus.
 *
 * @author  <NAME>
 * @since   10 December 2004
 * @author  <NAME>
 * @since   2 Jan 2005
 * @author  <NAME>
 * @since   9 February 2005
 * @version $LastChangedDate:2008-07-30 17:15:52 -0400 (Wed, 30 Jul 2008) $
 */
public class SuffixArray implements Corpus {

//===============================================================
// Constants
//===============================================================

	/**
	 * A random number generator used in the quick sort
	 * implementation.
	 */
	public static final Random RAND = new Random();
	
	
	/**
	 * The maximum length suffix to consider during sorting.
	 */
	public static int MAX_COMPARISON_LENGTH = 20;
	
	
	public static int INVERTED_INDEX_CAPACITY = 1000000;
	public static boolean POPULATE_INVERTED_INDEX = false;
	
	
	/** Logger for this class. */
	private static final Logger logger = 
		Logger.getLogger(SuffixArray.class.getName());
	
	
//===============================================================
// Member variables
//===============================================================

	protected int[] suffixes;
	protected CorpusArray corpus;
	
	protected InvertedIndex invertedIndex;
//===============================================================
// Constructor(s)
//===============================================================

	/** 
	 * Constructor takes a CorpusArray and creates a sorted
	 * suffix array from it.
	 */
{'fraction_non_alphanumeric': 0.18790115532734275, 'fraction_numerical': 0.02086007702182285, 'mean_word_length': 4.224643755238893, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6809373', 'n_tokens_mistral': 1734, 'n_tokens_neox': 1506, 'n_words': 703}