<commit_before>/**
 * This is the intermediate form of data that allows it to be serialized to multiple formats.
 */
package Util;

import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.UUID;

import Util.tokenformat.JSONRWFormat;
import Util.tokenformat.TokenRWFormat;

/**
 *  """
    An object that stores data in a form that can be serialized
    """
 * @author auerbach
 *
 */

public class StorageToken extends Serializable implements Iterable<String> {

	
	public static final String ID_KEY = "id";
	public static final String CLASS_ID_KEY = "classId";
	
	public static final Set<String> reservedKeys = new HashSet<>();
	
	private Map<String, Object> data;
	
	static
	{
		reservedKeys.add(CLASS_ID_KEY);
		reservedKeys.add(ID_KEY);
	}
	
	
	public StorageToken()
	{
	    super();
	    this.data = new HashMap<>();
	}
	
	
	public StorageToken(Map<String, Object> data, String id, String classId)
	{
		this.data = data;
		
		if(data == null)
			this.data = new HashMap<>();
		
		if(id != null)
		{
			this.setId(id);
		}
		
		else if(!this.data.containsKey(ID_KEY))
		{
			this.data.put(ID_KEY, UUID.randomUUID().toString());
		}
		
		if(classId != null)
		{
			this.setClassId(classId);
		}
		
		
		
	}
	
	//Accessors
	public String getId()
	{
		return (String) this.data.get(ID_KEY);
	}
	
	
	public void setId(String id)
	{
		this.data.put(ID_KEY, id);
	}
	<commit_msg>minor bug fix.  We don't want to nest storage tokens. when we serialize them.<commit_after>/**
 * This is the intermediate form of data that allows it to be serialized to multiple formats.
 */
package Util;

import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.UUID;

import Util.tokenformat.JSONRWFormat;
import Util.tokenformat.TokenRWFormat;

/**
 *  """
    An object that stores data in a form that can be serialized
    """
 * @author auerbach
 *
 */

public class StorageToken extends Serializable implements Iterable<String> {

	
	public static final String ID_KEY = "id";
	public static final String CLASS_ID_KEY = "classId";
	
	public static final Set<String> reservedKeys = new HashSet<>();
	
	private Map<String, Object> data;
	
	static
	{
		reservedKeys.add(CLASS_ID_KEY);
		reservedKeys.add(ID_KEY);
	}
	
	
	public StorageToken()
	{
	    super();
	    this.data = new HashMap<>();
	}
	
	
	public StorageToken(Map<String, Object> data, String id, String classId)
	{
		this.data = data;
		
		if(data == null)
			this.data = new HashMap<>();
		
		if(id != null)
		{
			this.setId(id);
		}
		
		else if(!this.data.containsKey(ID_KEY))
		{
			this.data.put(ID_KEY, UUID.randomUUID().toString());
		}
		
		if(classId != null)
		{
			this.setClassId(classId);
		}
		
		
		
	}
	
	//Serialization	
	public StorageToken saveToToken()
	{//Don't add an additional hashmap if it's already a storage token.
		return this;
	}
	
	//Accessors
	public String getId()
	{
		return (String) this.data.get(ID_KEY);
	}
	
	
	public void setId(String id)
	{
		this.data.put(ID_KEY, id);
	}
	
{'fraction_non_alphanumeric': 0.12276214833759591, 'fraction_numerical': 0.0, 'mean_word_length': 3.3218232044198897, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '221234', 'n_tokens_mistral': 1173, 'n_tokens_neox': 1041, 'n_words': 282}