<commit_before>/*******************************************************************************
* Copyright (c) 2009, 2012 Centrum Wiskunde en Informatica (CWI)
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
*    <NAME> - interfaces and implementation
*    <NAME> - labeled map types
*******************************************************************************/
package org.eclipse.imp.pdb.facts.io.binary;

import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;

import org.eclipse.imp.pdb.facts.IBool;
import org.eclipse.imp.pdb.facts.IConstructor;
import org.eclipse.imp.pdb.facts.IDateTime;
import org.eclipse.imp.pdb.facts.IInteger;
import org.eclipse.imp.pdb.facts.IList;
import org.eclipse.imp.pdb.facts.IMap;
import org.eclipse.imp.pdb.facts.INode;
import org.eclipse.imp.pdb.facts.IRational;
import org.eclipse.imp.pdb.facts.IReal;
import org.eclipse.imp.pdb.facts.ISet;
import org.eclipse.imp.pdb.facts.ISourceLocation;
import org.eclipse.imp.pdb.facts.IString;
import org.eclipse.imp.pdb.facts.ITuple;
import org.eclipse.imp.pdb.facts.IValue;
import org.eclipse.imp.pdb.facts.type.ITypeVisitor;
import org.eclipse.imp.pdb.facts.type.Type;
import org.eclipse.imp.pdb.facts.type.TypeStore;
import org.eclipse.imp.pdb.facts.util.IndexedSet;

// TODO Change this thing so it doesn't use recursion.
/**
 * @author <NAME>
 */
public class BinaryWriter{
	/*package*/ static final String CharEncoding = "UTF8";
	private final static int BOOL_HEADER = 0x01;
//	private final static int INTEGER_HEADER = 0x02;
	private final static int BIG_INTEGER_HEADER = 0x03; // Special case of INTEGER_HEADER (flags for alternate encoding).
	private final static int DOUBLE_HEADER = 0x04;
	private final static int STRING_HEADER = 0x05;
	private final static int SOURCE_LOCATION_HEADER = 0x06;
	private final static int DATE_TIME_HEADER = 0x10;
	private final static int TUPLE_HEADER = 0x07;
	private final static int NODE_HEADER = 0x08;
	private final static int ANNOTATED_NODE_HEADER = 0x09;
	private final static int CONSTRUCTOR_HEADER = 0x0a;
	private final static int ANNOTATED_CONSTRUCTOR_HEADER = 0x0b;
	private final static int LIST_HEADER = 0x0c;
	private final static int SET_HEADER = 0x0d;
	private final static int MAP_HEADER = 0x0f;
	private final static int RATIONAL_HEADER = 0x11;

	private final static int VALUE_TYPE_HEADER = 0x01;
	private final static int VOID_TYPE_HEADER = 0x02;
	private final static int BOOL_TYPE_HEADER = 0x03;
	private final static int INTEGER_TYPE_HEADER = 0x04;
	private final static int DOUBLE_TYPE_HEADER = 0x05;
	private final static int STRING_TYPE_HEADER = 0x06;
	private final static int SOURCE_LOCATION_TYPE_HEADER = 0x07;
	private final static int DATE_TIME_TYPE_HEADER = 0x14;
	private final static int NODE_TYPE_HEADER = 0x08;
	private final static int TUPLE_TYPE_HEADER = 0x09;
	private final static int LIST_TYPE_HEADER = 0x0a;
	private final static int SET_TYPE_HEADER = 0x0b;
	private final static int MAP_TYPE_HEADER = 0x0d;
	private final static int PARAMETER_TYPE_HEADER = 0x0e;
	private final static int ADT_TYPE_HEADER = 0x0f;
	private final static int CONSTRUCTOR_TYPE_HEADER = 0x10;
	private final static int ALIAS_TYPE_HEADER = 0x11;
	private final static int ANNOTATED_NODE_TYPE_HEADER = 0x12;
	private final static int ANNOTATED_CONSTRUCTOR_TYPE_HEADER = 0x13;
	private final static int RATIONAL_TYPE_HEADER = 0x15;
	private final static int NUM_TYPE_HEADER = 0x16;
	
	private final static int SHARED_FLAG = 0x80;
	private final static int TYPE_SHARED_FLAG = 0x40;
	private final static int URL_SHARED_FLAG = 0x20;
	private final static int NAME_SHARED_FLAG = 0x20;
	
	private final static int HAS_FIELD_NAMES = 0x20;
	
	private final static int DATE_TIME_INDICATOR = 0x01;
	private final static int DATE_INDICATOR = 0x02;
	private final static int TIME_INDICATOR = 0x03;
	
	private final IndexedSet<IValue> sharedValues;
	private final IndexedSet<Type> sharedTypes;
	private final IndexedSet<String> sharedPaths;
	private final IndexedSet<String> sharedNames;
	
	private final IValue value;
	private final OutputStream out;
	private final TypeStore typeStore;
	
	public BinaryWriter(IValue value, OutputStream outputStream, TypeStore typeStore){
		super();
		
		this.value = value;
		this.out = outputStream;
		this.typeStore = typeStore;
		
		sharedValues = new IndexedSet<>();
		sharedTypes = new IndexedSet<>();
		sharedPaths = new IndexedSet<>();
		sharedNames = new IndexedSet<>();
	}
	
	public void serialize() throws IOException{
		doSerialize(value);
	}
	
	private void doSerialize(IValue value) throws IOException{
		// This special cases the hashing logic: if we have a constructor with
		// at least one location annotation, don't try to hash it
		boolean tryHashing = true;
		
		if (value.getType().isAbstractData()) {
			IConstructor consValue = (IConstructor)value;
			if (consValue.asAnnotatable().hasAnnotations()) {
				Map<String,IValue> amap = consValue.asAnnotatable().getAnnotations();
				for (Entry<String, IValue> aEntry : amap.entrySet()) {
					Type aType = aEntry.getValue().getType();
					if (!aType.isBottom() && aType.isSourceLocation()) {
						tryHashing = false;
						break;
					}
				}
			}
		}
		
		if (tryHashing) {
			int valueId = sharedValues.get(value);
			if(valueId != -1){
				out.write(SHARED_FLAG);
				printInteger(valueId);
				return;
			}
		}
		
		// This sucks and is order dependent :-/.
		if(value instanceof IBool){
			writeBool((IBool) value);
		}else if(value instanceof IInteger){
			writeInteger((IInteger) value);
		}else if(value instanceof IRational){
			writeRational((IRational) value);
		}else if(value instanceof IReal){
			writeDouble((IReal) value);
		}else if(value instanceof IString){
			writeString((IString) value);
		}else if(value instanceof ISourceLocation){
			writeSourceLocation((ISourceLocation) value);
		}else if(value instanceof IDateTime){
			writeDateTime((IDateTime) value);
		}else if(value instanceof ITuple){
			writeTuple((ITuple) value);
		}else if(value instanceof IConstructor){
			IConstructor constructor = (IConstructor) value;
			if(!constructor.asAnnotatable().hasAnnotations()) writeConstructor(constructor);
			else writeAnnotatedConstructor(constructor);
		}else if(value instanceof INode){
			INode node = (INode) value;
			if(!node.asAnnotatable().hasAnnotations()) writeNode(node);
			else writeAnnotatedNode(node);
		}else if(value instanceof IList){
			writeList((IList) value);
		}else if(value instanceof ISet){
				writeSet((ISet) value);
		}else if(value instanceof IMap){
			writeMap((IMap) value);<commit_msg>fixed bug in binary writer<commit_after>/*******************************************************************************
* Copyright (c) 2009, 2012 Centrum Wiskunde en Informatica (CWI)
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
*    <NAME> - interfaces and implementation
*    <NAME> - labeled map types
*******************************************************************************/
package org.eclipse.imp.pdb.facts.io.binary;

import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;

import org.eclipse.imp.pdb.facts.IBool;
import org.eclipse.imp.pdb.facts.IConstructor;
import org.eclipse.imp.pdb.facts.IDateTime;
import org.eclipse.imp.pdb.facts.IInteger;
import org.eclipse.imp.pdb.facts.IList;
import org.eclipse.imp.pdb.facts.IMap;
import org.eclipse.imp.pdb.facts.INode;
import org.eclipse.imp.pdb.facts.IRational;
import org.eclipse.imp.pdb.facts.IReal;
import org.eclipse.imp.pdb.facts.ISet;
import org.eclipse.imp.pdb.facts.ISourceLocation;
import org.eclipse.imp.pdb.facts.IString;
import org.eclipse.imp.pdb.facts.ITuple;
import org.eclipse.imp.pdb.facts.IValue;
import org.eclipse.imp.pdb.facts.type.ITypeVisitor;
import org.eclipse.imp.pdb.facts.type.Type;
import org.eclipse.imp.pdb.facts.type.TypeStore;
import org.eclipse.imp.pdb.facts.util.IndexedSet;

// TODO Change this thing so it doesn't use recursion.
/**
 * @author <NAME>
 */
public class BinaryWriter{
	/*package*/ static final String CharEncoding = "UTF8";
	private final static int BOOL_HEADER = 0x01;
//	private final static int INTEGER_HEADER = 0x02;
	private final static int BIG_INTEGER_HEADER = 0x03; // Special case of INTEGER_HEADER (flags for alternate encoding).
	private final static int DOUBLE_HEADER = 0x04;
	private final static int STRING_HEADER = 0x05;
	private final static int SOURCE_LOCATION_HEADER = 0x06;
	private final static int DATE_TIME_HEADER = 0x10;
	private final static int TUPLE_HEADER = 0x07;
	private final static int NODE_HEADER = 0x08;
	private final static int ANNOTATED_NODE_HEADER = 0x09;
	private final static int CONSTRUCTOR_HEADER = 0x0a;
	private final static int ANNOTATED_CONSTRUCTOR_HEADER = 0x0b;
	private final static int LIST_HEADER = 0x0c;
	private final static int SET_HEADER = 0x0d;
	private final static int MAP_HEADER = 0x0f;
	private final static int RATIONAL_HEADER = 0x11;

	private final static int VALUE_TYPE_HEADER = 0x01;
	private final static int VOID_TYPE_HEADER = 0x02;
	private final static int BOOL_TYPE_HEADER = 0x03;
	private final static int INTEGER_TYPE_HEADER = 0x04;
	private final static int DOUBLE_TYPE_HEADER = 0x05;
	private final static int STRING_TYPE_HEADER = 0x06;
	private final static int SOURCE_LOCATION_TYPE_HEADER = 0x07;
	private final static int DATE_TIME_TYPE_HEADER = 0x14;
	private final static int NODE_TYPE_HEADER = 0x08;
	private final static int TUPLE_TYPE_HEADER = 0x09;
	private final static int LIST_TYPE_HEADER = 0x0a;
	private final static int SET_TYPE_HEADER = 0x0b;
	private final static int MAP_TYPE_HEADER = 0x0d;
	private final static int PARAMETER_TYPE_HEADER = 0x0e;
	private final static int ADT_TYPE_HEADER = 0x0f;
	private final static int CONSTRUCTOR_TYPE_HEADER = 0x10;
	private final static int ALIAS_TYPE_HEADER = 0x11;
	private final static int ANNOTATED_NODE_TYPE_HEADER = 0x12;
	private final static int ANNOTATED_CONSTRUCTOR_TYPE_HEADER = 0x13;
	private final static int RATIONAL_TYPE_HEADER = 0x15;
	private final static int NUM_TYPE_HEADER = 0x16;
	
	private final static int SHARED_FLAG = 0x80;
	private final static int TYPE_SHARED_FLAG = 0x40;
	private final static int URL_SHARED_FLAG = 0x20;
	private final static int NAME_SHARED_FLAG = 0x20;
	
	private final static int HAS_FIELD_NAMES = 0x20;
	
	private final static int DATE_TIME_INDICATOR = 0x01;
	private final static int DATE_INDICATOR = 0x02;
	private final static int TIME_INDICATOR = 0x03;
	
	private final IndexedSet<IValue> sharedValues;
	private final IndexedSet<Type> sharedTypes;
	private final IndexedSet<String> sharedPaths;
	private final IndexedSet<String> sharedNames;
	
	private final IValue value;
	private final OutputStream out;
	private final TypeStore typeStore;
	
	public BinaryWriter(IValue value, OutputStream outputStream, TypeStore typeStore){
		super();
		
		this.value = value;
		this.out = outputStream;
		this.typeStore = typeStore;
		
		sharedValues = new IndexedSet<>();
		sharedTypes = new IndexedSet<>();
		sharedPaths = new IndexedSet<>();
		sharedNames = new IndexedSet<>();
	}
	
	public void serialize() throws IOException{
		doSerialize(value);
	}
	
	private void doSerialize(IValue value) throws IOException{
		// This special cases the hashing logic: if we have a constructor with
		// at least one location annotation, don't try to hash it
		boolean tryHashing = true;
		
		if (value.getType().isAbstractData()) {
			IConstructor consValue = (IConstructor)value;
			if (consValue.asAnnotatable().hasAnnotations()) {
				Map<String,IValue> amap = consValue.asAnnotatable().getAnnotations();
				for (Entry<String, IValue> aEntry : amap.entrySet()) {
					Type aType = aEntry.getValue().getType();
					if (!aType.isBottom() && aType.isSourceLocation()) {
						tryHashing = false;
						break;
					}
				}
			}
		}
		
		if (tryHashing) {
			int valueId = sharedValues.get(value);
			if(valueId != -1){
				out.write(SHARED_FLAG);
				printInteger(valueId);
				return;
			}
		}
		
		// This sucks and is order dependent :-/.
		if(value instanceof IBool){
			writeBool((IBool) value);
		}else if(value instanceof IInteger){
			writeInteger((IInteger) value);
		}else if(value instanceof IRational){
			writeRational((IRational) value);
		}else if(value instanceof IReal){
			writeDouble((IReal) value);
		}else if(value instanceof IString){
			writeString((IString) value);
		}else if(value instanceof ISourceLocation){
			writeSourceLocation((ISourceLocation) value);
		}else if(value instanceof IDateTime){
			writeDateTime((IDateTime) value);
		}else if(value instanceof ITuple){
			writeTuple((ITuple) value);
		}else if(value instanceof IConstructor){
			IConstructor constructor = (IConstructor) value;
			if((constructor.mayHaveKeywordParameters() && constructor.asWithKeywordParameters().hasParameters())
			  || !constructor.asAnnotatable().hasAnnotations()) {
				writeConstructor(constructor);
			}
			else {
				writeAnnotatedConstructor(constructor);
			}
		}else if(value instanceof INode){
			INode node = (INode) value;
			if ((node.mayHaveKeywordParameters() && node.asWithKeywordParameters().hasParameters()) 
				|| !node.asAnnotatable().hasAnnotations()) {
				writeNode(node);
			}
			else {
				writeAnnotatedNode(node);
			}
		}else if(value instanceof IList){
			writeList((IList) value);
		}else if(value instanceof ISet){
				writeSet((ISet) value);
		}else if(value instanceof IMap){
			writeMap((IMap) value);
{'fraction_non_alphanumeric': 0.11277231952157198, 'fraction_numerical': 0.019792111633205184, 'mean_word_length': 4.862687813021703, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2366536', 'n_tokens_mistral': 4934, 'n_tokens_neox': 4337, 'n_words': 1242}