<commit_before>package com.psddev.dari.util;

import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.net.URLConnection;
import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
import java.nio.charset.IllegalCharsetNameException;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/** IO utility methods. */
public final class IoUtils {

    private static final int BUFFER_SIZE = 0x1000;
    private static final Logger LOGGER = LoggerFactory.getLogger(IoUtils.class);

    /**
     * Closes the given {@code closeable}.
     *
     * @param closeable If {@code null}, does nothing.
     * @param suppressError If {@code true}, logs the error instead of throwing it.
     */
    public static void close(Closeable closeable, boolean suppressError) throws IOException {
        if (closeable == null) {
            return;
        }

        try {
            closeable.close();
        } catch (IOException error) {
            if (suppressError) {
                if (LOGGER.isWarnEnabled()) {
                    LOGGER.warn("Can't close [" + closeable + "]!", error);
                }
            } else {
                throw error;
            }
        }
    }

    /**
     * Closes the given {@code closeable}, logging any errors that occur
     * instead of throwing them.
     *
     * @param closeable If {@code null}, does nothing.
     */
    public static void closeQuietly(Closeable closeable) {
        try {
            close(closeable, true);
        } catch (IOException error) {
            // This should never trigger with #close(suppressError = true).
        }
    }

    /**
     * Copies from the given {@code source} to the given {@code destination}.
     * Doesn't close either streams.
     *
     * @param source Can't be {@code null}.
     * @param destination Can't be {@code null}.
     * @return Number of bytes copied.
     */
    public static long copy(InputStream source, OutputStream destination) throws IOException {
        byte[] buffer = new byte[BUFFER_SIZE];
        long total = 0L;
        for (int read; (read = source.read(buffer)) > -1; ) {
            destination.write(buffer, 0, read);
            total += read;
        }
        return total;
    }

    /**
     * Copies the given {@code source} to the given {@code destination}.
     *
     * @param source Can't be {@code null}.
     * @param destination Can't be {@code null}.
     * @return Number of bytes copied.
     */
    public static long copy(File source, File destination) throws IOException {
        createFile(destination);

        FileInputStream sourceInput = new FileInputStream(source);

        try {
            FileChannel sourceChannel = sourceInput.getChannel();
            long total = sourceChannel.size();
            FileOutputStream destinationOutput = new FileOutputStream(destination);

            try {
                sourceChannel.transferTo(0, total, destinationOutput.getChannel());
                return total;

            } finally {
                close(destinationOutput, true);
            }

        } finally {
            closeQuietly(sourceInput);
        }
    }

    /**
     * Returns all bytes from the given {@code input}. Doesn't close
     * the stream.
     *
     * @param input Can't be {@code null}.
     * @return Never {@code null}.
     */
    public static byte[] toByteArray(InputStream input) throws IOException {
        ByteArrayOutputStream output = new ByteArrayOutputStream();
        copy(input, output);
        return output.toByteArray();
    }

    /**
     * Returns all bytes from the given {@code file}.
     *
     * @param file Can't be {@code null}.
     * @return Never {@code null}.
     */
    public static byte[] toByteArray(File file) throws IOException {
        InputStream input = new FileInputStream(file);
        try {
            return toByteArray(input);
        } finally {
            closeQuietly(input);
        }
    }
<commit_msg>Changes IoUtils#copy(File, File) to not use channels.<commit_after>package com.psddev.dari.util;

import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.Charset;
import java.nio.charset.IllegalCharsetNameException;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/** IO utility methods. */
public final class IoUtils {

    private static final int BUFFER_SIZE = 0x1000;
    private static final Logger LOGGER = LoggerFactory.getLogger(IoUtils.class);

    /**
     * Closes the given {@code closeable}.
     *
     * @param closeable If {@code null}, does nothing.
     * @param suppressError If {@code true}, logs the error instead of throwing it.
     */
    public static void close(Closeable closeable, boolean suppressError) throws IOException {
        if (closeable == null) {
            return;
        }

        try {
            closeable.close();
        } catch (IOException error) {
            if (suppressError) {
                if (LOGGER.isWarnEnabled()) {
                    LOGGER.warn("Can't close [" + closeable + "]!", error);
                }
            } else {
                throw error;
            }
        }
    }

    /**
     * Closes the given {@code closeable}, logging any errors that occur
     * instead of throwing them.
     *
     * @param closeable If {@code null}, does nothing.
     */
    public static void closeQuietly(Closeable closeable) {
        try {
            close(closeable, true);
        } catch (IOException error) {
            // This should never trigger with #close(suppressError = true).
        }
    }

    /**
     * Copies from the given {@code source} to the given {@code destination}.
     * Doesn't close either streams.
     *
     * @param source Can't be {@code null}.
     * @param destination Can't be {@code null}.
     * @return Number of bytes copied.
     */
    public static long copy(InputStream source, OutputStream destination) throws IOException {
        byte[] buffer = new byte[BUFFER_SIZE];
        long total = 0L;
        for (int read; (read = source.read(buffer)) > -1; ) {
            destination.write(buffer, 0, read);
            total += read;
        }
        return total;
    }

    /**
     * Copies the given {@code source} to the given {@code destination}.
     *
     * @param source Can't be {@code null}.
     * @param destination Can't be {@code null}.
     * @return Number of bytes copied.
     */
    public static long copy(File source, File destination) throws IOException {
        createFile(destination);

        FileInputStream sourceInput = new FileInputStream(source);

        try {
            FileOutputStream destinationOutput = new FileOutputStream(destination);

            try {
                return copy(sourceInput, destinationOutput);

            } finally {
                destinationOutput.close();
            }

        } finally {
            sourceInput.close();
        }
    }

    /**
     * Returns all bytes from the given {@code input}. Doesn't close
     * the stream.
     *
     * @param input Can't be {@code null}.
     * @return Never {@code null}.
     */
    public static byte[] toByteArray(InputStream input) throws IOException {
        ByteArrayOutputStream output = new ByteArrayOutputStream();
        copy(input, output);
        return output.toByteArray();
    }

    /**
     * Returns all bytes from the given {@code file}.
     *
     * @param file Can't be {@code null}.
     * @return Never {@code null}.
     */
    public static byte[] toByteArray(File file) throws IOException {
        InputStream input = new FileInputStream(file);
        try {
            return toByteArray(input);
        } finally {
            closeQuietly(input);
        }
    }

{'fraction_non_alphanumeric': 0.10311545510079413, 'fraction_numerical': 0.002565668906536347, 'mean_word_length': 2.1363984674329504, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5007376', 'n_tokens_mistral': 2198, 'n_tokens_neox': 2126, 'n_words': 671}