<commit_before>package com.jmatio.io;


import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Collection;
import java.util.zip.DataFormatException;
import java.util.zip.Deflater;

import com.jmatio.common.MatDataTypes;
import com.jmatio.types.MLArray;
import com.jmatio.types.MLCell;
import com.jmatio.types.MLChar;
import com.jmatio.types.MLDouble;
import com.jmatio.types.MLSparse;
import com.jmatio.types.MLStructure;

/**
 * MAT-file writer.
 * 
 * Usage:
 * <pre><code>
 * //1. First create example arrays
 * double[] src = new double[] { 1.0, 2.0, 3.0, 4.0, 5.0, 6.0 };
 * MLDouble mlDouble = new MLDouble( "double_arr", src, 3 );
 * MLChar mlChar = new MLChar( "char_arr", "I am dummy" );
 *         
 * //2. write arrays to file
 * ArrayList<MLArray> list = new ArrayList<MLArray>();
 * list.add( mlDouble );
 * list.add( mlChar );
 * 
 * new MatFileWriter( "mat_file.mat", list );
 * </code></pre>
 * 
 * this is "equal" to Matlab commands:
 * <pre><code>
 * >> double_arr = [ 1 2; 3 4; 5 6];
 * >> char_arr = 'I am dummy';
 * >>
 * >> save('mat_file.mat', 'double_arr', 'char_arr');
 * </pre></code>
 * 
 * @author <NAME> (<a href="mailto:<EMAIL>"><EMAIL></a>)
 */
public class MatFileWriter
{
//    private static final Logger logger = Logger.getLogger(MatFileWriter.class);
    
    /**
     * Writes MLArrays into file given by <code>fileName</code>.
     * 
     * @param fileName - name of ouput file
     * @param data - <code>Collection</code> of <code>MLArray</code> elements
     * @throws IOException
     * @throws DataFormatException
     */
    public MatFileWriter(String fileName, Collection<MLArray> data) throws IOException
    {
        this( new File(fileName), data );
    }
    /**
     * Writes MLArrays into <code>File</code>.
     * 
     * @param file - an output <code>File</code>
     * @param data - <code>Collection</code> of <code>MLArray</code> elements
     * @throws IOException
     * @throws DataFormatException
     */
    public MatFileWriter(File file, Collection<MLArray> data) throws IOException
    {
        this( new DataOutputStream(new FileOutputStream(file)), data );
    }
    /**
     * Writes MLArrays into <code>OuputSteram</code>.
     * 
     * Writes MAT-file header and compressed data (<code>miCOMPRESSED</code>).
     * 
     * @param output - <code>OutputStream</code>
     * @param data - <code>Collection</code> of <code>MLArray</code> elements
     * @throws IOException
     */
    public MatFileWriter(DataOutputStream output, Collection<MLArray> data) throws IOException
    {
        //write header
        writeHeader(output);
        
        //write data
        for ( MLArray matrix : data )
        {
            //prepare buffer for MATRIX data
            ByteArrayOutputStream baos = new ByteArrayOutputStream();
            DataOutputStream dos = new DataOutputStream( baos );
            //write MATRIX bytes into buffer
            writeMatrix( dos, matrix );
            
            //compress data to save storage
            Deflater compresser = new Deflater();
            compresser.setInput(baos.toByteArray());
            compresser.finish();
            
            ByteArrayOutputStream compressed = new ByteArrayOutputStream();
            
            byte[] buffer = new byte[128];
            int toread = 0;
            do
            {
                toread = compresser.deflate(buffer);
                compressed.write(buffer);
            }
            while ( toread > 0 );
            
            //write COMPRESSED tag and compressed data into output stream
            output.writeInt( MatDataTypes.miCOMPRESSED );
            output.writeInt(compressed.size());
            output.write(compressed.toByteArray());
        }
        
        output.close();        
    }
    
    /**
     * Writes MAT-file header into <code>OutputStream</code>
     * @param os <code>OutputStream</code>
     * @throws IOException
     */
    private void writeHeader(DataOutputStream os) throws IOException
    {
        //write descriptive text
        MatFileHeader header = MatFileHeader.createHeader();
        char[] dest = new char[116];
        char[] src = header.getDescription().toCharArray();
        System.arraycopy(src, 0, dest, 0, src.length);
        for ( int i = 0; i < dest.length; i++ )
        {
            os.writeByte( dest[i] );
        }
        //write subsyst data offset
        os.write( new byte[8] );
        
        //write version
        int version = header.getVersion();
        os.writeByte(version >> 8);
        os.writeByte(version);
        
        os.write( header.getEndianIndicator() );
    }
    
    /**
     * Writes MATRIX into <code>OutputStream</code>.
     * 
     * @param os - <code>OutputStream</code>
     * @param array - a <code>MLArray</code>
     * @throws IOException
     */
    private void writeMatrix(DataOutputStream output, MLArray array) throws IOException
    {   
        OSArrayTag tag;
        ByteArrayOutputStream buffer;         
        DataOutputStream bufferDOS;
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        DataOutputStream dos = new DataOutputStream(baos);
        
        //flags
        writeFlags(dos, array);

        //dimensions
        writeDimensions(dos, array);
        
        //array name
        writeName(dos, array);
        <commit_msg>+<NAME>: modified jmatio slightly to use nio package to output to file<commit_after>package com.jmatio.io;


import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.WritableByteChannel;
import java.util.Collection;
import java.util.zip.DataFormatException;
import java.util.zip.Deflater;

import com.jmatio.common.MatDataTypes;
import com.jmatio.types.MLArray;
import com.jmatio.types.MLCell;
import com.jmatio.types.MLChar;
import com.jmatio.types.MLDouble;
import com.jmatio.types.MLSparse;
import com.jmatio.types.MLStructure;

/**
 * MAT-file writer.
 * 
 * Usage:
 * <pre><code>
 * //1. First create example arrays
 * double[] src = new double[] { 1.0, 2.0, 3.0, 4.0, 5.0, 6.0 };
 * MLDouble mlDouble = new MLDouble( "double_arr", src, 3 );
 * MLChar mlChar = new MLChar( "char_arr", "I am dummy" );
 *         
 * //2. write arrays to file
 * ArrayList<MLArray> list = new ArrayList<MLArray>();
 * list.add( mlDouble );
 * list.add( mlChar );
 * 
 * new MatFileWriter( "mat_file.mat", list );
 * </code></pre>
 * 
 * this is "equal" to Matlab commands:
 * <pre><code>
 * >> double_arr = [ 1 2; 3 4; 5 6];
 * >> char_arr = 'I am dummy';
 * >>
 * >> save('mat_file.mat', 'double_arr', 'char_arr');
 * </pre></code>
 * 
 * @author <NAME> (<a href="mailto:<EMAIL>"><EMAIL></a>)
 */
public class MatFileWriter
{
//    private static final Logger logger = Logger.getLogger(MatFileWriter.class);
    
    /**
     * Writes MLArrays into file given by <code>fileName</code>.
     * 
     * @param fileName - name of ouput file
     * @param data - <code>Collection</code> of <code>MLArray</code> elements
     * @throws IOException
     * @throws DataFormatException
     */
    public MatFileWriter(String fileName, Collection<MLArray> data) throws IOException
    {
        this( new File(fileName), data );
    }
    /**
     * Writes MLArrays into <code>File</code>.
     * 
     * @param file - an output <code>File</code>
     * @param data - <code>Collection</code> of <code>MLArray</code> elements
     * @throws IOException
     * @throws DataFormatException
     */
    public MatFileWriter(File file, Collection<MLArray> data) throws IOException
    {
        this( new FileOutputStream(file).getChannel(), data );
    }
    /**
     * Writes MLArrays into <code>OuputSteram</code>.
     * 
     * Writes MAT-file header and compressed data (<code>miCOMPRESSED</code>).
     * 
     * @param output - <code>OutputStream</code>
     * @param data - <code>Collection</code> of <code>MLArray</code> elements
     * @throws IOException
     */
    public MatFileWriter(WritableByteChannel channel, Collection<MLArray> data) throws IOException
    {
        //write header
        writeHeader(channel);
        
        //write data
        for ( MLArray matrix : data )
        {
            //prepare buffer for MATRIX data
            ByteArrayOutputStream baos = new ByteArrayOutputStream();
            DataOutputStream dos = new DataOutputStream( baos );
            //write MATRIX bytes into buffer
            writeMatrix( dos, matrix );
            
            //compress data to save storage
            Deflater compresser = new Deflater();
            compresser.setInput(baos.toByteArray());
            compresser.finish();
            
            ByteArrayOutputStream compressed = new ByteArrayOutputStream();
            
            byte[] buffer = new byte[128];
            int toread = 0;
            do
            {
                toread = compresser.deflate(buffer);
                compressed.write(buffer);
            }
            while ( toread > 0 );
            
            //write COMPRESSED tag and compressed data into output channel
            byte[] compressedBytes = compressed.toByteArray();
            ByteBuffer buf = ByteBuffer.allocateDirect(2 * 4 /* Int size */ + compressedBytes.length);
            buf.putInt( MatDataTypes.miCOMPRESSED );
            buf.putInt(compressedBytes.length);
            buf.put(compressedBytes);
            
            buf.flip();
            channel.write(buf);
        }
        
        channel.close();        
    }
    
    /**
     * Writes MAT-file header into <code>OutputStream</code>
     * @param os <code>OutputStream</code>
     * @throws IOException
     */
    private void writeHeader(WritableByteChannel channel) throws IOException
    {
        //write descriptive text
        MatFileHeader header = MatFileHeader.createHeader();
        char[] dest = new char[116];
        char[] src = header.getDescription().toCharArray();
        System.arraycopy(src, 0, dest, 0, src.length);
        
        byte[] endianIndicator = header.getEndianIndicator();
        
        ByteBuffer buf = ByteBuffer.allocateDirect(dest.length * 2 /* Char size */ + 2 + endianIndicator.length);
        
        for ( int i = 0; i < dest.length; i++ )
        {
            buf.put( (byte)dest[i] );
        }
        //write subsyst data offset
        buf.position( buf.position() + 8);
        
        //write version
        int version = header.getVersion();
        buf.put( (byte)(version >> 8) );
        buf.put( (byte)version );
        
        buf.put( endianIndicator );
        
        buf.flip();
        channel.write(buf);
    }
    
    /**
     * Writes MATRIX into <code>OutputStream</code>.
     * 
     * @param os - <code>OutputStream</code>
     * @param array - a <code>MLArray</code>
     * @throws IOException
     */
    private void writeMatrix(DataOutputStream output, MLArray array) throws IOException
    {   
        OSArrayTag tag;
        ByteArrayOutputStream buffer;         
        DataOutputStream bufferDOS;
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        DataOutputStream dos = new DataOutputStream(baos);
        
        //flags
        writeFlags(dos, array);

        //dimensions
        writeDimensions(dos, array);
        
        //array name
        writeName(dos, array);
        
{'fraction_non_alphanumeric': 0.11987218239917091, 'fraction_numerical': 0.006218153553847482, 'mean_word_length': 2.218454697053919, 'pattern_counts': {'":': 0, '<': 110, '<?xml version=': 0, '>': 130, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1985506', 'n_tokens_mistral': 3417, 'n_tokens_neox': 3304, 'n_words': 825}