<commit_before>use std::io::{self, Read};
use std::fs::File;
use std::path::Path;

use response::{Response, ResponseWriter};

pub struct FileLoader {
    ///The size, in bytes, of the file chunks. Default is 1024.
    pub chunk_size: usize
}

impl FileLoader {
    pub fn new() -> FileLoader {
        FileLoader {
            chunk_size: 1024
        }
    }

    pub fn send_file<'a, 'b, P: AsRef<Path>>(&self, path: P, response: Response<'a, 'b>) -> Result<(), Error<'a, 'b>> {
        let mut file = match File::open(path) {
            Ok(file) => file,
            Err(e) => return Err(Error::Open(e, response))
        };

        let mut writer = response.into_writer();
        let mut buffer = vec![0; self.chunk_size];
        loop {
            match file.read(&mut buffer) {
                Ok(len) if len == 0 => break,
                Ok(len) => writer.send(&buffer[..len]),
                Err(e) => return Err(Error::Read(e, writer))
            }
        }

        Ok(())
    }
}

pub enum Error<'a, 'b> {
    Open(io::Error, Response<'a, 'b>),
    Read(io::Error, ResponseWriter<'a, 'b>)
}

impl<'a, 'b> Error<'a, 'b> {
    pub fn io_error(&self) -> &io::Error {
        match *self {
            Error::Open(ref e, _) => e,
            Error::Read(ref e, _) => e
        }
    }
}<commit_msg>Increase file chunk size in FileLoader<commit_after>use std::io::{self, Read};
use std::fs::File;
use std::path::Path;

use response::{Response, ResponseWriter};

pub struct FileLoader {
    ///The size, in bytes, of the file chunks. Default is 1048576 (1 megabyte).
    pub chunk_size: usize
}

impl FileLoader {
    pub fn new() -> FileLoader {
        FileLoader {
            chunk_size: 1048576
        }
    }

    pub fn send_file<'a, 'b, P: AsRef<Path>>(&self, path: P, response: Response<'a, 'b>) -> Result<(), Error<'a, 'b>> {
        let mut file = match File::open(path) {
            Ok(file) => file,
            Err(e) => return Err(Error::Open(e, response))
        };

        let mut writer = response.into_writer();
        let mut buffer = vec![0; self.chunk_size];
        loop {
            match file.read(&mut buffer) {
                Ok(len) if len == 0 => break,
                Ok(len) => writer.send(&buffer[..len]),
                Err(e) => return Err(Error::Read(e, writer))
            }
        }

        Ok(())
    }
}

pub enum Error<'a, 'b> {
    Open(io::Error, Response<'a, 'b>),
    Read(io::Error, ResponseWriter<'a, 'b>)
}

impl<'a, 'b> Error<'a, 'b> {
    pub fn io_error(&self) -> &io::Error {
        match *self {
            Error::Open(ref e, _) => e,
            Error::Read(ref e, _) => e
        }
    }
}
{'fraction_non_alphanumeric': 0.17303370786516853, 'fraction_numerical': 0.010112359550561797, 'mean_word_length': 2.0560640732265445, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6915860', 'n_tokens_mistral': 910, 'n_tokens_neox': 897, 'n_words': 231}