<commit_before>/* Copyright (C) 2002 Univ. of Massachusetts Amherst, Computer Science Dept.
   This file is part of "MALLET" (MAchine Learning for LanguagE Toolkit).
   http://www.cs.umass.edu/~mccallum/mallet
   This software is provided under the terms of the Common Public License,
   version 1.0, as published by http://www.opensource.org.  For further
   information, see the file `LICENSE' included with this distribution. */




/** 
    @author <NAME> <a href="mailto:<EMAIL>"><EMAIL></a>
*/

package mstparser;

import java.util.ArrayList;
import java.io.*;
import java.util.Iterator;

public class Alphabet implements Serializable
{
    gnu.trove.TObjectIntHashMap map;
    int numEntries;
    boolean growthStopped = false;

    public Alphabet (int capacity)
    {
	this.map = new gnu.trove.TObjectIntHashMap (capacity);
	//this.map.setDefaultValue(-1);

	numEntries = 0;
    }

    public Alphabet ()
    {
	this (10000);
    }

	
    /** Return -1 if entry isn't present. */
    public int lookupIndex (Object entry)
    {
	if (entry == null) {
	    throw new IllegalArgumentException ("Can't lookup \"null\" in an Alphabet.");
	}

	int ret = map.get(entry);

	// Trove used to return -1 when a map didn't have an entry,
	// now it returns 0. (The following line used to have the
	// condition "ret == -1".) (JMB, 16-JAN-07)
	//if (ret == 0 && !growthStopped && addIfNotPresent) {
	if (ret == -1 && !growthStopped) {
	    ret = numEntries;
	    map.put (entry, ret);
	    numEntries++;
	}
	
	//System.out.println(entry + " :: " + ret);

	return ret;<commit_msg>Upgraded to newer Trove version.<commit_after>/* Copyright (C) 2002 Univ. of Massachusetts Amherst, Computer Science Dept.
   This file is part of "MALLET" (MAchine Learning for LanguagE Toolkit).
   http://www.cs.umass.edu/~mccallum/mallet
   This software is provided under the terms of the Common Public License,
   version 1.0, as published by http://www.opensource.org.  For further
   information, see the file `LICENSE' included with this distribution. */




/** 
    @author <NAME> <a href="mailto:<EMAIL>"><EMAIL></a>
*/

package mstparser;

import java.util.ArrayList;
import java.io.*;
import java.util.Iterator;

public class Alphabet implements Serializable
{
    gnu.trove.TObjectIntHashMap map;
    int numEntries;
    boolean growthStopped = false;

    public Alphabet (int capacity)
    {
	this.map = new gnu.trove.TObjectIntHashMap (capacity);
	//this.map.setDefaultValue(-1);

	numEntries = 0;
    }

    public Alphabet ()
    {
	this (10000);
    }

	
    /** Return -1 if entry isn't present. */
    public int lookupIndex (Object entry)
    {
	if (entry == null) {
	    throw new IllegalArgumentException ("Can't lookup \"null\" in an Alphabet.");
	}

	int ret = map.get(entry);

	if (ret == -1 && !growthStopped) {
	    ret = numEntries;
	    map.put (entry, ret);
	    numEntries++;
	}
	
	//System.out.println(entry + " :: " + ret);

	return ret;
{'fraction_non_alphanumeric': 0.12957937584803256, 'fraction_numerical': 0.012890094979647219, 'mean_word_length': 3.078838174273859, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7355886', 'n_tokens_mistral': 1015, 'n_tokens_neox': 936, 'n_words': 298}