<commit_before>/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package nz.co.gregs.dbvolution;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 *
 * Implements the abstractions necessary to handle arbitrary byte streams and
 * files stored in the database
 *
 * @author gregory.graham
 */
public class DBByteArray extends DBLargeObject {

    public static final long serialVersionUID = 1;
    byte[] bytes;

    public DBByteArray(Object object) {
        super(object);
    }

    public DBByteArray() {
        super();
    }

    @Override
    public String getSQLDatatype() {
        return "BLOB";
    }

    @Override
    protected void setFromResultSet(ResultSet resultSet, String fullColumnName) throws SQLException {
        InputStream input = new BufferedInputStream(resultSet.getBinaryStream(fullColumnName));
        List<byte[]> byteArrays = new ArrayList<byte[]>();

        int totalBytesRead = 0;
        try {
            byte[] resultSetBytes;
            resultSetBytes = new byte[100000];
            int bytesRead = input.read(resultSetBytes);
            while (bytesRead > 0) {
                totalBytesRead = totalBytesRead + bytesRead;
                byteArrays.add(resultSetBytes);
                resultSetBytes = new byte[100000];
                bytesRead = input.read(resultSetBytes);
            }
        } catch (IOException ex) {
            Logger.getLogger(DBByteArray.class.getName()).log(Level.SEVERE, null, ex);
        }

        bytes = new byte[totalBytesRead];
        int bytesAdded = 0;
        for (byte[] someBytes : byteArrays) {
            System.arraycopy(someBytes, 0, bytes, bytesAdded, Math.min(someBytes.length, bytes.length));
            bytesAdded += someBytes.length;
        }

        this.setValue(bytes);
//        this.useEqualsOperator(resultSet.getBytes(fullColumnName));
    }

    @Override
    public String getSQLValue() {
        throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
    }

    public byte[] readFromFileSystem(String originalFile) throws FileNotFoundException, IOException {
        File file = new File(originalFile);
        return readFromFileSystem(file);
    }

    public byte[] readFromFileSystem(File originalFile) throws FileNotFoundException, IOException {
        System.out.println("FILE: " + originalFile.getAbsolutePath());
        bytes = new byte[(int) originalFile.length()];
        InputStream input = null;
        try {
            int totalBytesRead = 0;
            input = new BufferedInputStream(new FileInputStream(originalFile));
            while (totalBytesRead < bytes.length) {
                int bytesRemaining = bytes.length - totalBytesRead;
                //input.read() returns -1, 0, or more :
                int bytesRead = input.read(bytes, totalBytesRead, bytesRemaining);
                if (bytesRead > 0) {
                    totalBytesRead = totalBytesRead + bytesRead;
                }
            }
            /*
             the above style is a bit tricky: it places bytes into the 'result' array;
             'result' is an output parameter;
             the while loop usually has a single iteration only.
             */
        } finally {
            if (input != null) {
                input.close();
            }
        }
        return bytes;
    }

    public void writeToFileSystem(String originalFile) throws FileNotFoundException, IOException {
        File file = new File(originalFile);
        file.createNewFile();
        writeToFileSystem(file);
    }

    public void writeToFileSystem(File originalFile) throws FileNotFoundException, IOException {
        if (bytes != null && originalFile != null) {
            System.out.println("FILE: " + originalFile.getAbsolutePath());
            OutputStream output = null;
            try {
                output = new BufferedOutputStream(new FileOutputStream(originalFile));
                output.write(this.bytes);
                output.flush();
                output.close();
                output = null;
            } finally {
                if (output != null) {
                    output.close();
                }
            }
        }
    }

    @Override
    public InputStream getInputStream() {
        return new BufferedInputStream(new ByteArrayInputStream(bytes));
    }
}
<commit_msg>Improved the safety of writeToFileSystem() slightly<commit_after>/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package nz.co.gregs.dbvolution;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 *
 * Implements the abstractions necessary to handle arbitrary byte streams and
 * files stored in the database
 *
 * @author gregory.graham
 */
public class DBByteArray extends DBLargeObject {

    public static final long serialVersionUID = 1;
    byte[] bytes;

    public DBByteArray(Object object) {
        super(object);
    }

    public DBByteArray() {
        super();
    }

    @Override
    public String getSQLDatatype() {
        return "BLOB";
    }

    @Override
    protected void setFromResultSet(ResultSet resultSet, String fullColumnName) throws SQLException {
        InputStream input = new BufferedInputStream(resultSet.getBinaryStream(fullColumnName));
        List<byte[]> byteArrays = new ArrayList<byte[]>();

        int totalBytesRead = 0;
        try {
            byte[] resultSetBytes;
            resultSetBytes = new byte[100000];
            int bytesRead = input.read(resultSetBytes);
            while (bytesRead > 0) {
                totalBytesRead = totalBytesRead + bytesRead;
                byteArrays.add(resultSetBytes);
                resultSetBytes = new byte[100000];
                bytesRead = input.read(resultSetBytes);
            }
        } catch (IOException ex) {
            Logger.getLogger(DBByteArray.class.getName()).log(Level.SEVERE, null, ex);
        }

        bytes = new byte[totalBytesRead];
        int bytesAdded = 0;
        for (byte[] someBytes : byteArrays) {
            System.arraycopy(someBytes, 0, bytes, bytesAdded, Math.min(someBytes.length, bytes.length));
            bytesAdded += someBytes.length;
        }

        this.setValue(bytes);
//        this.useEqualsOperator(resultSet.getBytes(fullColumnName));
    }

    @Override
    public String getSQLValue() {
        throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
    }

    public byte[] readFromFileSystem(String originalFile) throws FileNotFoundException, IOException {
        File file = new File(originalFile);
        return readFromFileSystem(file);
    }

    public byte[] readFromFileSystem(File originalFile) throws FileNotFoundException, IOException {
        System.out.println("FILE: " + originalFile.getAbsolutePath());
        bytes = new byte[(int) originalFile.length()];
        InputStream input = null;
        try {
            int totalBytesRead = 0;
            input = new BufferedInputStream(new FileInputStream(originalFile));
            while (totalBytesRead < bytes.length) {
                int bytesRemaining = bytes.length - totalBytesRead;
                //input.read() returns -1, 0, or more :
                int bytesRead = input.read(bytes, totalBytesRead, bytesRemaining);
                if (bytesRead > 0) {
                    totalBytesRead = totalBytesRead + bytesRead;
                }
            }
            /*
             the above style is a bit tricky: it places bytes into the 'result' array;
             'result' is an output parameter;
             the while loop usually has a single iteration only.
             */
        } finally {
            if (input != null) {
                input.close();
            }
        }
        return bytes;
    }

    public void writeToFileSystem(String originalFile) throws FileNotFoundException, IOException {
        File file = new File(originalFile);
        writeToFileSystem(file);
    }

    public void writeToFileSystem(File originalFile) throws FileNotFoundException, IOException {
        if (bytes != null && originalFile != null) {
            System.out.println("FILE: " + originalFile.getAbsolutePath());
            if (!originalFile.exists()){
                originalFile.createNewFile();
            }
            OutputStream output = null;
            try {
                output = new BufferedOutputStream(new FileOutputStream(originalFile));
                output.write(this.bytes);
                output.flush();
                output.close();
                output = null;
            } finally {
                if (output != null) {
                    output.close();
                }
            }
        }
    }

    @Override
    public InputStream getInputStream() {
        return new BufferedInputStream(new ByteArrayInputStream(bytes));
    }
}

{'fraction_non_alphanumeric': 0.08917197452229299, 'fraction_numerical': 0.0041156295933365995, 'mean_word_length': 2.0575194727381665, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6341282', 'n_tokens_mistral': 2627, 'n_tokens_neox': 2507, 'n_words': 659}