<commit_before>package org.vitrivr.cineast.core.decode.m3d;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.joml.Vector3f;
import org.joml.Vector3i;
import org.vitrivr.cineast.core.config.DecoderConfig;
import org.vitrivr.cineast.core.data.m3d.Mesh;
import org.vitrivr.cineast.core.decode.general.Decoder;
import org.vitrivr.cineast.core.util.LogHelper;

import java.io.*;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;

/**
 * Decodes STereoLithography (.stl) files and returns a Mesh representation. Requires
 * JOML to work properly.
 *
 * @author rgasser
 * @version 1.0
 * @created 29.12.16
 */
public class STLMeshDecoder implements Decoder<Mesh> {
    /** Default logging facility. */
    private static final Logger LOGGER = LogManager.getLogger();

    /** HashSet containing all the mime-types supported by this ImageDecoder instance. */
    private static final Set<String> supportedFiles;
    static {
        HashSet<String> tmp = new HashSet<>();
        tmp.add("application/3d-stl");
        supportedFiles = Collections.unmodifiableSet(tmp);
    }

    /** Path to the input file. */
    private Path inputFile;

    /** Flag indicating whether or not the Decoder is done decoding and the content has been obtained. */
    private AtomicBoolean complete = new AtomicBoolean(false);

    /**
     * Initializes the decoder with a file. This is a necessary step before content can be retrieved from
     * the decoder by means of the getNext() method.
     *
     * @param path   Path to the file that should be decoded.
     * @param config DecoderConfiguration used by the decoder.
     * @return True if initialization was successful, false otherwise.
     */
    @Override
    public boolean init(Path path, DecoderConfig config) {
        this.inputFile = path;
        this.complete.set(false);
        return true;
    }

    /**
     * Fetches the next piece of content of type T and returns it. This method can be safely invoked until
     * complete() returns false. From which on this method will return null.
     *
     * @return Content of type T.
     */
    @Override
    public Mesh getNext() {
        try {
            InputStream is = Files.newInputStream(this.inputFile);
            byte[] header = new byte[6];
            if (is.read(header) == 6) {
                if ((new String(header)).contains("solid ")) {
                    LOGGER.info("Found term 'solid' in header. Treating the STL file as ASCII STL file!");
                    return this.readAscii(is);
                } else {
                    LOGGER.info("Did not find term 'solid' in header. Treating the STL file as binary STL file!");
                    return this.readBinary(is, 74);
                }
            } else {
                LOGGER.warn("Could not read the first 10 bytes of the file {}. This is probably not an STL file.", this.inputFile.toString());
                return Mesh.EMPTY;
            }
        } catch (IOException e) {
            LOGGER.error("Could not decode STL file {} due to an IO exception ({})", this.inputFile.toString(), LogHelper.getStackTrace(e));
            this.complete.set(true);
            return Mesh.EMPTY;
        } finally {
            this.complete.set(true);
        }
    }

    /**
     * Reads an ASCII STL file.
     *
     * @param is InputStream to read from.
     * @return Mesh
     * @throws IOException If an error occurs during reading.
     */
    private Mesh readAscii(InputStream is)  throws IOException {
        BufferedReader br = new BufferedReader(new InputStreamReader(is));
        String line = null;
        int idx = 0;
        Mesh mesh = new Mesh(100,100);
        while ((line = br.readLine()) != null && !line.startsWith("endsolid")) {
            if (line.startsWith("facet normal ")) {
                String[] splitNormal = line.split("\\s+");
                while (!(line = br.readLine()).equals("endfacet")) {
                    String[] splitVertex = line.split("\\s+");
                    if (line.startsWith("vertex")) {
                        mesh.addVertex(new Vector3f(Float.parseFloat(splitVertex[2]),Float.parseFloat(splitVertex[3]), Float.parseFloat(splitVertex[4])));
                    }
                }

                 /* Add a new face to the Mesh. */
                mesh.addFace(new Vector3i(3*idx + 1, 3*idx + 2, 3*idx + 3));

                /* Increment index. */
                idx += 1;
            }
        }

        /* Close the buffered reader. */
        br.close();

        /* This covers the case, where the file starts with 'solid ' but is not an ASCII file. Unfortunately, such files do exist. */
        if (mesh.numberOfVertices() == 0) {
            LOGGER.warn("The provided ASCII STL file does not seem to contain any normals or vertices. Trying to decode it as binary STL even though it was marked as being ASCII.");
            InputStream newIs = Files.newInputStream(this.inputFile);
            return this.readBinary(newIs, 80);
        } else {
            return mesh;
        }
    }

    /**
     * Reads a binary STL file.
     *
     * @param is InputStream to read from.
     * @param skip Number of bytes to skip before reading the STL file.
     * @return Mesh
     * @throws IOException If an error occurs during reading.
     */
    private Mesh readBinary(InputStream is, int skip) throws IOException {
        /* Prepare a ByteBuffer to read the rest of the STL file. */
        byte[] bytes = new byte[48];
        ByteBuffer buffer = ByteBuffer.wrap(bytes);
        buffer.order(ByteOrder.LITTLE_ENDIAN);

        /* Skip the STL header! */
        is.skip(skip);

        /* Read the bytes for the size (unsigned 32 bit int, little-endian). */
        byte[] sizeBytes = new byte[4];
        is.read(sizeBytes, 0, 4);
        long triangles = ((sizeBytes[0] & 0xFF)) | ((sizeBytes[1] & 0xFF) << 8) | ((sizeBytes[2] & 0xFF) << 16) | ((sizeBytes[3] & 0xFF) << 24);
        if (triangles <= 0) return null;

        Mesh mesh = new Mesh((int)triangles, (int)triangles * 3);

        /* Now add all triangles. */
        for (int i=0; i<triangles; i++) {
            /* Ready 48 bytes from the stream. */
            buffer.rewind();
            is.read(bytes);

            float nx = buffer.getFloat();
            nx = buffer.getFloat();
            nx = buffer.getFloat();

            /* Add the vertices and the vertex-normal to the mesh. */
            mesh.addVertex(new Vector3f(buffer.getFloat(), buffer.getFloat(), buffer.getFloat()));
            mesh.addVertex(new Vector3f(buffer.getFloat(), buffer.getFloat(), buffer.getFloat()));
            mesh.addVertex(new Vector3f(buffer.getFloat(), buffer.getFloat(), buffer.getFloat()));

            /* Add a new face to the Mesh. */
            if (!mesh.addFace(new Vector3i(3 * i, 3 * i + 1, 3 * i + 2))) {
                LOGGER.warn("Could not add face {}/{}/{} because index points to non-existing vertex.", 3*i, 3*i + 1, 3*i + 2);
            }<commit_msg>Case where number of triangles in STL file exceeds a certain limit is now handled.<commit_after>package org.vitrivr.cineast.core.decode.m3d;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.joml.Vector3f;
import org.joml.Vector3i;
import org.vitrivr.cineast.core.config.DecoderConfig;
import org.vitrivr.cineast.core.data.m3d.Mesh;
import org.vitrivr.cineast.core.decode.general.Decoder;
import org.vitrivr.cineast.core.util.LogHelper;

import java.io.*;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;

/**
 * Decodes STereoLithography (.stl) files and returns a Mesh representation. Requires
 * JOML to work properly.
 *
 * @author rgasser
 * @version 1.0
 * @created 29.12.16
 */
public class STLMeshDecoder implements Decoder<Mesh> {
    /** Default logging facility. */
    private static final Logger LOGGER = LogManager.getLogger();

    /** HashSet containing all the mime-types supported by this ImageDecoder instance. */
    private static final Set<String> supportedFiles;
    static {
        HashSet<String> tmp = new HashSet<>();
        tmp.add("application/3d-stl");
        supportedFiles = Collections.unmodifiableSet(tmp);
    }

    /** Path to the input file. */
    private Path inputFile;

    /** Flag indicating whether or not the Decoder is done decoding and the content has been obtained. */
    private AtomicBoolean complete = new AtomicBoolean(false);

    /**
     * Initializes the decoder with a file. This is a necessary step before content can be retrieved from
     * the decoder by means of the getNext() method.
     *
     * @param path   Path to the file that should be decoded.
     * @param config DecoderConfiguration used by the decoder.
     * @return True if initialization was successful, false otherwise.
     */
    @Override
    public boolean init(Path path, DecoderConfig config) {
        this.inputFile = path;
        this.complete.set(false);
        return true;
    }

    /**
     * Fetches the next piece of content of type T and returns it. This method can be safely invoked until
     * complete() returns false. From which on this method will return null.
     *
     * @return Content of type T.
     */
    @Override
    public Mesh getNext() {
        try {
            InputStream is = Files.newInputStream(this.inputFile);
            byte[] header = new byte[6];
            if (is.read(header) == 6) {
                if ((new String(header)).contains("solid ")) {
                    LOGGER.info("Found term 'solid' in header. Treating the STL file as ASCII STL file!");
                    return this.readAscii(is);
                } else {
                    LOGGER.info("Did not find term 'solid' in header. Treating the STL file as binary STL file!");
                    return this.readBinary(is, 74);
                }
            } else {
                LOGGER.warn("Could not read the first 10 bytes of the file {}. This is probably not an STL file.", this.inputFile.toString());
                return Mesh.EMPTY;
            }
        } catch (IOException e) {
            LOGGER.error("Could not decode STL file {} due to an IO exception ({})", this.inputFile.toString(), LogHelper.getStackTrace(e));
            this.complete.set(true);
            return Mesh.EMPTY;
        } finally {
            this.complete.set(true);
        }
    }

    /**
     * Reads an ASCII STL file.
     *
     * @param is InputStream to read from.
     * @return Mesh
     * @throws IOException If an error occurs during reading.
     */
    private Mesh readAscii(InputStream is)  throws IOException {
        BufferedReader br = new BufferedReader(new InputStreamReader(is));
        String line = null;
        int idx = 0;
        Mesh mesh = new Mesh(100,100);
        while ((line = br.readLine()) != null && !line.startsWith("endsolid")) {
            if (line.startsWith("facet normal ")) {
                String[] splitNormal = line.split("\\s+");
                while (!(line = br.readLine()).equals("endfacet")) {
                    String[] splitVertex = line.split("\\s+");
                    if (line.startsWith("vertex")) {
                        mesh.addVertex(new Vector3f(Float.parseFloat(splitVertex[2]),Float.parseFloat(splitVertex[3]), Float.parseFloat(splitVertex[4])));
                    }
                }

                 /* Add a new face to the Mesh. */
                mesh.addFace(new Vector3i(3*idx + 1, 3*idx + 2, 3*idx + 3));

                /* Increment index. */
                idx += 1;
            }
        }

        /* Close the buffered reader. */
        br.close();

        /* This covers the case, where the file starts with 'solid ' but is not an ASCII file. Unfortunately, such files do exist. */
        if (mesh.numberOfVertices() == 0) {
            LOGGER.warn("The provided ASCII STL file does not seem to contain any normals or vertices. Trying to decode it as binary STL even though it was marked as being ASCII.");
            InputStream newIs = Files.newInputStream(this.inputFile);
            return this.readBinary(newIs, 80);
        } else {
            return mesh;
        }
    }

    /**
     * Reads a binary STL file.
     *
     * @param is InputStream to read from.
     * @param skip Number of bytes to skip before reading the STL file.
     * @return Mesh
     * @throws IOException If an error occurs during reading.
     */
    private Mesh readBinary(InputStream is, int skip) throws IOException {
        /* Prepare a ByteBuffer to read the rest of the STL file. */
        byte[] bytes = new byte[48];
        ByteBuffer buffer = ByteBuffer.wrap(bytes);
        buffer.order(ByteOrder.LITTLE_ENDIAN);

        /* Skip the STL header! */
        is.skip(skip);

        /* Read the bytes for the size (unsigned 32 bit int, little-endian). */
        byte[] sizeBytes = new byte[4];
        is.read(sizeBytes, 0, 4);
        long triangles = ((sizeBytes[0] & 0xFF)) | ((sizeBytes[1] & 0xFF) << 8) | ((sizeBytes[2] & 0xFF) << 16) | ((sizeBytes[3] & 0xFF) << 24);

        /* TODO: Properly handle models whose triangles > Integer.MAX_VALUE/3. */
        if (triangles <= 0 ) {
            LOGGER.error("The number of triangles in the Mesh seems to be smaller than zero. This STL file is probably corrupt!");
            return null;
        } else if (triangles > Integer.MAX_VALUE/3) {
            LOGGER.error("The number of triangles in the Mesh exceeds the limit that can currently be processed by STLMeshDecoder.");
            return null;
        }

        Mesh mesh = new Mesh((int)triangles, (int)triangles * 3);

        /* Now add all triangles. */
        for (int i=0; i<triangles; i++) {
            /* Ready 48 bytes from the stream. */
            buffer.rewind();
            is.read(bytes);

            float nx = buffer.getFloat();
            nx = buffer.getFloat();
            nx = buffer.getFloat();

            /* Add the vertices and the vertex-normal to the mesh. */
            mesh.addVertex(new Vector3f(buffer.getFloat(), buffer.getFloat(), buffer.getFloat()));
            mesh.addVertex(new Vector3f(buffer.getFloat(), buffer.getFloat(), buffer.getFloat()));
            mesh.addVertex(new Vector3f(buffer.getFloat(), buffer.getFloat(), buffer.getFloat()));

            /* Add a new face to the Mesh. */
            if (!mesh.addFace(new Vector3i(3 * i, 3 * i + 1, 3 * i + 2))) {
                LOGGER.warn("Could not add face {}/{}/{} because index points to non-existing vertex.", 3*i, 3*i + 1, 3*i + 2);
            }
{'fraction_non_alphanumeric': 0.11424183748414235, 'fraction_numerical': 0.01108366161447553, 'mean_word_length': 2.174650275540483, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1562261', 'n_tokens_mistral': 4399, 'n_tokens_neox': 4150, 'n_words': 1353}