<commit_before>
/*
 * Carrot2 project.
 *
 * Copyright (C) 2002-2006, <NAME>, <NAME>.
 * Portions (C) Contributors listed in "carrot2.CONTRIBUTORS" file.
 * All rights reserved.
 *
 * Refer to the full license file "carrot2.LICENSE"
 * in the root folder of the repository checkout or at:
 * http://www.carrot2.org/carrot2.LICENSE
 */

package com.dawidweiss.carrot.core.local.linguistic.tokens;

import java.util.*;

/**
 * A dynamically constructed token sequence.
 * 
 * @author <NAME>
 * @version $Revision$
 */
public class MutableTokenSequence implements TokenSequence
{
    /** Tokens */
    private List tokens;
    
    /** String image of this MutableTokenSequence */
    private String image;

    /**
     * Creates an empty MutableTokenSequence.
     */
    public MutableTokenSequence()
    {
        tokens = new ArrayList();
    }

    /**
     * Creates a MutableTokenSequence with one initial token
     * 
     * @param token
     */
    public MutableTokenSequence(Token token)
    {
        this();
        addToken(token);
    }

    /**
     * Creates a MutableTokenSequence with some initial tokens
     * 
     * @param tokens
     */
    public MutableTokenSequence(Token [] tokens)
    {
        this();
        addTokens(tokens);
    }

    /**
     * @param token
     */
    public void addToken(Token token)
    {
        tokens.add(token);
    }

    /**
     * @param tokenArray
     */
    public void addTokens(Token [] tokenArray)
    {
        tokens.addAll(Arrays.asList(tokenArray));
    }

    /**
     * @param index
     * @param token
     */
    public void setTokenAt(int index, Token token)
    {
        tokens.set(index, token);
    }

    /*
     * (non-Javadoc)
     * 
     * @see com.dawidweiss.carrot.core.local.linguistic.tokens.TokenSequence#getLength()
     */
    public int getLength()
    {
        return tokens.size();
    }

    /*
     * (non-Javadoc)
     * 
     * @see com.dawidweiss.carrot.core.local.linguistic.tokens.TokenSequence#getTokenAt(int)
     */
    public Token getTokenAt(int index)
    {
        return (Token) tokens.get(index);
    }

    /*
     * (non-Javadoc)
     * 
     * @see com.dawidweiss.carrot.core.local.linguistic.tokens.TokenSequence#copyTo(com.dawidweiss.carrot.core.local.linguistic.tokens.Token[],
     *      int, int, int)
     */
    public int copyTo(Token [] destination, int startAt,
        int destinationStartAt, int maxLength)
    {
        throw new RuntimeException("Not implemented yet");
    }

    /*
     * (non-Javadoc)
     * 
     * @see java.lang.Object#equals(java.lang.Object)
     */
    public boolean equals(Object obj)
    {
        if (obj == this)
        {
            return true;
        }

        if (obj == null)
        {
            return false;
        }

        if (!(obj instanceof TokenSequence))
        {
            return false;
        }
        else
        {
            TokenSequence tokenSequence = (TokenSequence) obj;
            if (tokenSequence.getLength() != tokens.size())
            {
                return false;
            }
            else
            {
                for (int t = 0; t < tokens.size(); t++)
                {
                    if (!tokens.get(t).equals(tokenSequence.getTokenAt(t)))
                    {
                        return false;
                    }
                }
            }
        }

        return true;
    }

    /*
     * (non-Javadoc)
     * 
     * @see java.lang.Object#hashCode()
     */
    public int hashCode()
    {
        int hash = 0;

        // As hash code is supposed to be fast - we calculate it for at most 8
        // first tokens
        for (int t = 0; t < tokens.size() && t < 8; t++)
        {
            if (tokens.get(t) != null)
            {
                hash += tokens.get(t).hashCode();
            }
        }

        return hash;
    }

    /*
     * (non-Javadoc)
     * 
     * @see java.lang.Object#toString()
     */
    public String toString()
    {
        if (image == null)
        {
            StringBuffer stringBuffer = new StringBuffer();
            for (Iterator tokensIter = tokens.iterator(); tokensIter.hasNext();)
            {
                Token token = (Token) tokensIter.next();
                token.appendTo(stringBuffer);
                stringBuffer.append(" ");
            }
            
            image = stringBuffer.toString();
        }

        return image;
    }
}<commit_msg>- made one field final. - removed empty javadocs.<commit_after>
/*
 * Carrot2 project.
 *
 * Copyright (C) 2002-2006, <NAME>, <NAME>.
 * Portions (C) Contributors listed in "carrot2.CONTRIBUTORS" file.
 * All rights reserved.
 *
 * Refer to the full license file "carrot2.LICENSE"
 * in the root folder of the repository checkout or at:
 * http://www.carrot2.org/carrot2.LICENSE
 */

package com.dawidweiss.carrot.core.local.linguistic.tokens;

import java.util.*;

/**
 * A dynamically constructed token sequence.
 * 
 * @author <NAME>
 * @version $Revision$
 */
public class MutableTokenSequence implements TokenSequence
{
    /** Tokens */
    private final List tokens;
    
    /** String image of this MutableTokenSequence */
    private String image;

    /**
     * Creates an empty MutableTokenSequence.
     */
    public MutableTokenSequence()
    {
        tokens = new ArrayList();
    }

    /**
     * Creates a MutableTokenSequence with one initial token
     * 
     * @param token
     */
    public MutableTokenSequence(Token token)
    {
        this();
        addToken(token);
    }

    /**
     * Creates a MutableTokenSequence with some initial tokens
     * 
     * @param tokens
     */
    public MutableTokenSequence(Token [] tokens)
    {
        this();
        addTokens(tokens);
    }

    public void addToken(Token token)
    {
        tokens.add(token);
    }

    public void addTokens(Token [] tokenArray)
    {
        tokens.addAll(Arrays.asList(tokenArray));
    }

    public void setTokenAt(int index, Token token)
    {
        tokens.set(index, token);
    }

    public int getLength()
    {
        return tokens.size();
    }

    public Token getTokenAt(int index)
    {
        return (Token) tokens.get(index);
    }

    public int copyTo(Token [] destination, int startAt,
        int destinationStartAt, int maxLength)
    {
        throw new RuntimeException("Not implemented yet");
    }

    public boolean equals(Object obj)
    {
        if (obj == this)
        {
            return true;
        }

        if (obj == null)
        {
            return false;
        }

        if (!(obj instanceof TokenSequence))
        {
            return false;
        }
        else
        {
            TokenSequence tokenSequence = (TokenSequence) obj;
            if (tokenSequence.getLength() != tokens.size())
            {
                return false;
            }
            else
            {
                for (int t = 0; t < tokens.size(); t++)
                {
                    if (!tokens.get(t).equals(tokenSequence.getTokenAt(t)))
                    {
                        return false;
                    }
                }
            }
        }

        return true;
    }

    public int hashCode()
    {
        int hash = 0;

        // As hash code is supposed to be fast - we calculate it for at most 8
        // first tokens
        for (int t = 0; t < tokens.size() && t < 8; t++)
        {
            if (tokens.get(t) != null)
            {
                hash += tokens.get(t).hashCode();
            }
        }

        return hash;
    }

    public String toString()
    {
        if (image == null)
        {
            StringBuffer stringBuffer = new StringBuffer();
            for (Iterator tokensIter = tokens.iterator(); tokensIter.hasNext();)
            {
                Token token = (Token) tokensIter.next();
                token.appendTo(stringBuffer);
                stringBuffer.append(" ");
            }
            
            image = stringBuffer.toString();
        }

        return image;
    }
}
{'fraction_non_alphanumeric': 0.11181121818631085, 'fraction_numerical': 0.004447739065974796, 'mean_word_length': 1.530478274460769, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1122715', 'n_tokens_mistral': 2372, 'n_tokens_neox': 2268, 'n_words': 549}