<commit_before>package org.bimserver.serializers.binarygeometry;

/******************************************************************************
 * Copyright (C) 2009-2014  BIMserver.org
 * 
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *****************************************************************************/

import java.io.DataOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
import java.nio.IntBuffer;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.bimserver.emf.IdEObject;
import org.bimserver.models.geometry.GeometryData;
import org.bimserver.models.geometry.GeometryInfo;
import org.bimserver.plugins.serializers.AbstractGeometrySerializer;
import org.bimserver.plugins.serializers.AligningOutputStream;
import org.bimserver.plugins.serializers.SerializerException;
import org.eclipse.emf.ecore.EClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class BinaryGeometrySerializer extends AbstractGeometrySerializer {
	private static final Logger LOGGER = LoggerFactory.getLogger(BinaryGeometrySerializer.class);
	private static final byte FORMAT_VERSION = 4;
	private static final byte GEOMETRY_TYPE_TRIANGLES = 0;
	private static final byte GEOMETRY_TYPE_INSTANCE = 1;

	@Override
	public void reset() {
		setMode(Mode.BODY);
	}

	@Override
	protected boolean write(OutputStream outputStream) throws SerializerException {
		if (getMode() == Mode.BODY) {
			try {
				calculateGeometryExtents();
				writeGeometries(outputStream);
			} catch (Exception e) {
				LOGGER.error("", e);
			}
			setMode(Mode.FINISHED);
			return true;
		} else if (getMode() == Mode.FINISHED) {
			return false;
		}
		return false;
	}

	private void writeGeometries(OutputStream outputStream) throws IOException {
		long start = System.nanoTime();

		DataOutputStream dataOutputStream = new DataOutputStream(outputStream);
		dataOutputStream.writeUTF("BGS");
		dataOutputStream.writeByte(FORMAT_VERSION);
		
		Bounds modelBounds = new Bounds();
		int nrObjects = 0;
		
		EClass productClass = getModel().getPackageMetaData().getEClass("IfcProduct");
		
		List<IdEObject> products = getModel().getAllWithSubTypes(productClass);
		
		for (IdEObject ifcProduct : products) {
			GeometryInfo geometryInfo = (GeometryInfo) ifcProduct.eGet(ifcProduct.eClass().getEStructuralFeature("geometry"));
			if (geometryInfo != null && geometryInfo.getTransformation() != null) {
				Bounds objectBounds = new Bounds(new Float3(geometryInfo.getMinBounds().getX(), geometryInfo.getMinBounds().getY(), geometryInfo.getMinBounds()
						.getZ()), new Float3(geometryInfo.getMaxBounds().getX(), geometryInfo.getMaxBounds().getY(), geometryInfo.getMaxBounds().getZ()));
				modelBounds.integrate(objectBounds);
				nrObjects++;
			}
		}
		modelBounds.writeTo(dataOutputStream);
		dataOutputStream.writeInt(nrObjects);
		int bytesSaved = 0;
		int bytesTotal = 0;
		
		Set<Long> concreteGeometrySent = new HashSet<>();
		
		dataOutputStream.flush();
		
		int counter = 0;
		
		for (IdEObject ifcProduct : products) {
			GeometryInfo geometryInfo = (GeometryInfo) ifcProduct.eGet(ifcProduct.eClass().getEStructuralFeature("geometry"));
			if (geometryInfo != null && geometryInfo.getTransformation() != null) {
				dataOutputStream.writeUTF(ifcProduct.eClass().getName());
				dataOutputStream.writeLong(ifcProduct.getOid());

				GeometryData geometryData = geometryInfo.getData();
				byte[] vertices = geometryData.getVertices();
				
				// BEWARE, ByteOrder is always LITTLE_ENDIAN, because that's what GPU's seem to prefer, Java's ByteBuffer default is BIG_ENDIAN though!
				
				bytesTotal += vertices.length;
				byte geometryType = concreteGeometrySent.contains(geometryData.getOid()) ? GEOMETRY_TYPE_INSTANCE : GEOMETRY_TYPE_TRIANGLES;
				dataOutputStream.write(geometryType);
				
				if (outputStream instanceof AligningOutputStream) {
					((AligningOutputStream)outputStream).align4();
				} else {
					int skip = 4 - (dataOutputStream.size() % 4);
					if(skip != 0 && skip != 4) {
						dataOutputStream.write(new byte[skip]);
					}
				}
				
				dataOutputStream.write(geometryInfo.getTransformation());
				
				if (concreteGeometrySent.contains(geometryData.getOid())) {
					dataOutputStream.writeLong(geometryData.getOid());
					bytesSaved += vertices.length;
				} else {
					ByteBuffer vertexByteBuffer = ByteBuffer.wrap(vertices);
					dataOutputStream.writeLong(geometryData.getOid());
					
					Bounds objectBounds = new Bounds(geometryInfo.getMinBounds(), geometryInfo.getMaxBounds());
					objectBounds.writeTo(dataOutputStream);
					
					ByteBuffer indicesBuffer = ByteBuffer.wrap(geometryData.getIndices());
					dataOutputStream.writeInt(indicesBuffer.capacity() / 4);
					dataOutputStream.write(indicesBuffer.array());
					
					dataOutputStream.writeInt(vertexByteBuffer.capacity() / 4);
					dataOutputStream.write(vertexByteBuffer.array());
					
					ByteBuffer normalsBuffer = ByteBuffer.wrap(geometryData.getNormals());
					dataOutputStream.writeInt(normalsBuffer.capacity() / 4);
					dataOutputStream.write(normalsBuffer.array());
					
					if (geometryData.getMaterialIndices() != null) {
						ByteBuffer materialIndexByteBuffer = ByteBuffer.wrap(geometryData.getMaterialIndices());
						materialIndexByteBuffer.order(ByteOrder.LITTLE_ENDIAN);
						IntBuffer materialIndicesBuffer = materialIndexByteBuffer.asIntBuffer();
						ByteBuffer materialsByteBuffer = ByteBuffer.wrap(geometryData.getMaterials());
						materialsByteBuffer.order(ByteOrder.LITTLE_ENDIAN);
						FloatBuffer materialsBuffer = materialsByteBuffer.asFloatBuffer();
						dataOutputStream.writeInt(materialIndicesBuffer.capacity() * 3);
						for (int i=0; i<materialIndicesBuffer.capacity(); i++) {
							int materialIndex = materialIndicesBuffer.get(i);
							if (materialIndex == -1) {
								ByteBuffer test = ByteBuffer.wrap(new byte[16]);
								test.order(ByteOrder.LITTLE_ENDIAN);
								FloatBuffer fl = test.asFloatBuffer();
								fl.put(1f);
								fl.put(0f);
								fl.put(0f);
								fl.put(1f);
								for (int j=0; j<3; j++) {
									dataOutputStream.write(test.array());
								}
							} else {
								float r = materialsBuffer.get(materialIndex);
								float g = materialsBuffer.get(materialIndex + 1);
								float b = materialsBuffer.get(materialIndex + 2);
								float a = materialsBuffer.get(materialIndex + 3);
								ByteBuffer test = ByteBuffer.wrap(new byte[16]);
								test.order(ByteOrder.LITTLE_ENDIAN);
								FloatBuffer fl = test.asFloatBuffer();
								fl.put(r);
								fl.put(g);
								fl.put(b);
								fl.put(a);
								for (int j=0; j<3; j++) {
									dataOutputStream.write(test.array());
								}
							}
						}
					} else {
						dataOutputStream.writeInt(0);
					}
					
					concreteGeometrySent.add(geometryData.getOid());
				}
				counter++;
				if (counter % 12 == 0) {
					dataOutputStream.flush();
				}
			}
		}
		dataOutputStream.flush();
		if (bytesTotal != 0 && bytesSaved != 0) {
			LOGGER.debug((100 * bytesSaved / bytesTotal) + "% saved");
		}
		long end = System.nanoTime();
		LOGGER.debug(((end - start) / 1000000) + " ms");
	}
}
<commit_msg>Added a bit of documentation<commit_after>package org.bimserver.serializers.binarygeometry;

/******************************************************************************
 * Copyright (C) 2009-2014  BIMserver.org
 * 
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *****************************************************************************/

import java.io.DataOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
import java.nio.IntBuffer;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.bimserver.emf.IdEObject;
import org.bimserver.models.geometry.GeometryData;
import org.bimserver.models.geometry.GeometryInfo;
import org.bimserver.plugins.serializers.AbstractGeometrySerializer;
import org.bimserver.plugins.serializers.AligningOutputStream;
import org.bimserver.plugins.serializers.SerializerException;
import org.eclipse.emf.ecore.EClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class BinaryGeometrySerializer extends AbstractGeometrySerializer {
	private static final Logger LOGGER = LoggerFactory.getLogger(BinaryGeometrySerializer.class);
	private static final byte FORMAT_VERSION = 4;
	private static final byte GEOMETRY_TYPE_TRIANGLES = 0;
	private static final byte GEOMETRY_TYPE_INSTANCE = 1;

	@Override
	public void reset() {
		setMode(Mode.BODY);
	}

	@Override
	protected boolean write(OutputStream outputStream) throws SerializerException {
		if (getMode() == Mode.BODY) {
			try {
				calculateGeometryExtents();
				writeGeometries(outputStream);
			} catch (Exception e) {
				LOGGER.error("", e);
			}
			setMode(Mode.FINISHED);
			return true;
		} else if (getMode() == Mode.FINISHED) {
			return false;
		}
		return false;
	}

	private void writeGeometries(OutputStream outputStream) throws IOException {
		long start = System.nanoTime();

		DataOutputStream dataOutputStream = new DataOutputStream(outputStream);
		// Identifier for clients to determine if this server is even serving binary geometry
		dataOutputStream.writeUTF("BGS");
		
		// Version of the current format being outputted, should be changed for every (released) change in protocol 
		dataOutputStream.writeByte(FORMAT_VERSION);
		
		Bounds modelBounds = new Bounds();
		int nrObjects = 0;
		
		// All access to EClass is being done generically to support multiple IFC schema's with 1 serializer
		EClass productClass = getModel().getPackageMetaData().getEClass("IfcProduct");
		
		List<IdEObject> products = getModel().getAllWithSubTypes(productClass);
		
		// First iteration, to determine number of objects with geometry and calculate model bounds
		for (IdEObject ifcProduct : products) {
			GeometryInfo geometryInfo = (GeometryInfo) ifcProduct.eGet(ifcProduct.eClass().getEStructuralFeature("geometry"));
			if (geometryInfo != null && geometryInfo.getTransformation() != null) {
				Bounds objectBounds = new Bounds(new Float3(geometryInfo.getMinBounds().getX(), geometryInfo.getMinBounds().getY(), geometryInfo.getMinBounds()
						.getZ()), new Float3(geometryInfo.getMaxBounds().getX(), geometryInfo.getMaxBounds().getY(), geometryInfo.getMaxBounds().getZ()));
				modelBounds.integrate(objectBounds);
				nrObjects++;
			}
		}
		modelBounds.writeTo(dataOutputStream);
		dataOutputStream.writeInt(nrObjects);
		int bytesSaved = 0;
		int bytesTotal = 0;
		
		// Keeping track of geometry already sent, this can be used for instancing of reused geometry
		Set<Long> concreteGeometrySent = new HashSet<>();
		
		// Flushing here so the client can show progressbar etc...
		dataOutputStream.flush();
		
		int counter = 0;
		
		// Second iteration actually writing the geometry
		for (IdEObject ifcProduct : products) {
			GeometryInfo geometryInfo = (GeometryInfo) ifcProduct.eGet(ifcProduct.eClass().getEStructuralFeature("geometry"));
			if (geometryInfo != null && geometryInfo.getTransformation() != null) {
				dataOutputStream.writeUTF(ifcProduct.eClass().getName());
				dataOutputStream.writeLong(ifcProduct.getOid());

				GeometryData geometryData = geometryInfo.getData();
				byte[] vertices = geometryData.getVertices();
				
				// BEWARE, ByteOrder is always LITTLE_ENDIAN, because that's what GPU's seem to prefer, Java's ByteBuffer default is BIG_ENDIAN though!
				
				bytesTotal += vertices.length;
				byte geometryType = concreteGeometrySent.contains(geometryData.getOid()) ? GEOMETRY_TYPE_INSTANCE : GEOMETRY_TYPE_TRIANGLES;
				dataOutputStream.write(geometryType);
				
				// This is an ugly hack to align the bytes, but for 2 different kinds of output (this first one is the websocket implementation)
				if (outputStream instanceof AligningOutputStream) {
					((AligningOutputStream)outputStream).align4();
				} else {
					int skip = 4 - (dataOutputStream.size() % 4);
					if(skip != 0 && skip != 4) {
						dataOutputStream.write(new byte[skip]);
					}
				}
				
				dataOutputStream.write(geometryInfo.getTransformation());
				
				if (concreteGeometrySent.contains(geometryData.getOid())) {
					// Reused geometry, only send the id of the reused geometry data
					dataOutputStream.writeLong(geometryData.getOid());
					bytesSaved += vertices.length;
				} else {
					ByteBuffer vertexByteBuffer = ByteBuffer.wrap(vertices);
					dataOutputStream.writeLong(geometryData.getOid());
					
					Bounds objectBounds = new Bounds(geometryInfo.getMinBounds(), geometryInfo.getMaxBounds());
					objectBounds.writeTo(dataOutputStream);
					
					ByteBuffer indicesBuffer = ByteBuffer.wrap(geometryData.getIndices());
					dataOutputStream.writeInt(indicesBuffer.capacity() / 4);
					dataOutputStream.write(indicesBuffer.array());
					
					dataOutputStream.writeInt(vertexByteBuffer.capacity() / 4);
					dataOutputStream.write(vertexByteBuffer.array());
					
					ByteBuffer normalsBuffer = ByteBuffer.wrap(geometryData.getNormals());
					dataOutputStream.writeInt(normalsBuffer.capacity() / 4);
					dataOutputStream.write(normalsBuffer.array());
					
					// Only when materials are used we send them
					if (geometryData.getMaterialIndices() != null) {
						ByteBuffer materialIndexByteBuffer = ByteBuffer.wrap(geometryData.getMaterialIndices());
						materialIndexByteBuffer.order(ByteOrder.LITTLE_ENDIAN);
						IntBuffer materialIndicesBuffer = materialIndexByteBuffer.asIntBuffer();
						ByteBuffer materialsByteBuffer = ByteBuffer.wrap(geometryData.getMaterials());
						materialsByteBuffer.order(ByteOrder.LITTLE_ENDIAN);
						FloatBuffer materialsBuffer = materialsByteBuffer.asFloatBuffer();
						dataOutputStream.writeInt(materialIndicesBuffer.capacity() * 3);
						for (int i=0; i<materialIndicesBuffer.capacity(); i++) {
							int materialIndex = materialIndicesBuffer.get(i);
							if (materialIndex == -1) {
								sendMaterial(dataOutputStream, 0, 1, 0, 1);
							} else {
								sendMaterial(dataOutputStream, materialsBuffer.get(materialIndex), materialsBuffer.get(materialIndex + 1), materialsBuffer.get(materialIndex + 2), materialsBuffer.get(materialIndex + 3));
							}
						}
					} else {
						// No materials used
						dataOutputStream.writeInt(0);
					}
					
					concreteGeometrySent.add(geometryData.getOid());
				}
				counter++;
				if (counter % 12 == 0) {
					// Flushing in batches, this is to limit the amount of WebSocket messages
					dataOutputStream.flush();
				}
			}
		}
		dataOutputStream.flush();
		if (bytesTotal != 0 && bytesSaved != 0) {
			LOGGER.debug((100 * bytesSaved / bytesTotal) + "% saved");
		}
		long end = System.nanoTime();
		LOGGER.debug(((end - start) / 1000000) + " ms");
	}

	private void sendMaterial(DataOutputStream dataOutputStream, float r, float g, float b, float a) throws IOException {
		ByteBuffer test = ByteBuffer.wrap(new byte[16]);
		test.order(ByteOrder.LITTLE_ENDIAN);
		FloatBuffer fl = test.asFloatBuffer();
		fl.put(r);
		fl.put(g);
		fl.put(b);
		fl.put(a);
		for (int j=0; j<3; j++) {
			dataOutputStream.write(test.array());
		}
	}
}

{'fraction_non_alphanumeric': 0.12418104225521429, 'fraction_numerical': 0.0073330528340445995, 'mean_word_length': 4.084963325183374, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7155210', 'n_tokens_mistral': 5707, 'n_tokens_neox': 4575, 'n_words': 1105}