<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use net_traits::{LoadData, Metadata, ProgressMsg};
use net_traits::ProgressMsg::{Payload, Done};
use mime_classifier::MIMEClassifier;
use resource_task::{start_sending, start_sending_sniffed};

use std::borrow::ToOwned;
use std::io;
use std::fs::File;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::mpsc::Sender;
use util::task::spawn_named;

static READ_SIZE: usize = 8192;

enum ReadStatus {
    Partial(Vec<u8>),
    EOF,
}

fn read_block(reader: &mut io::Read) -> Result<ReadStatus, String> {
    let mut buf = vec![0; READ_SIZE];
    match reader.read(buf.as_mut_slice()) {
        Ok(0) => return Ok(ReadStatus::EOF),
        Ok(n) => {
            buf.truncate(n);
            Ok(ReadStatus::Partial(buf))
        }
        Err(e) => Err(e.description().to_string()),
    }
}

fn read_all(reader: &mut io::Read, progress_chan: &Sender<ProgressMsg>)
            -> Result<(), String> {
    loop {
        match try!(read_block(reader)) {
            ReadStatus::Partial(buf) => progress_chan.send(Payload(buf)).unwrap(),
            ReadStatus::EOF => return Ok(()),
        }
    }
}

pub fn factory(load_data: LoadData, classifier: Arc<MIMEClassifier>) {
    let url = load_data.url;
    let start_chan = load_data.consumer;
    assert!(&*url.scheme == "file");
    spawn_named("file_loader".to_owned(), move || {
        let metadata = Metadata::default(url.clone());
        let file_path: Result<PathBuf, ()> = url.to_file_path();
        match file_path {
            Ok(file_path) => {
                match File::open(&file_path) {
                    Ok(ref mut reader) => {
                        let res = read_block(reader);
                        let (res, progress_chan) = match res {
                            Ok(ReadStatus::Partial(buf)) => {
                                let progress_chan = start_sending_sniffed(start_chan, metadata,
                                                                          classifier, &buf);<commit_msg>Auto merge of #5631 - Ms2ger:virtual-file, r=jdm<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use net_traits::{LoadData, Metadata, ProgressMsg};
use net_traits::ProgressMsg::{Payload, Done};
use mime_classifier::MIMEClassifier;
use resource_task::{start_sending, start_sending_sniffed};

use std::borrow::ToOwned;
use std::fs::File;
use std::io::Read;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::mpsc::Sender;
use util::task::spawn_named;

static READ_SIZE: usize = 8192;

enum ReadStatus {
    Partial(Vec<u8>),
    EOF,
}

fn read_block(reader: &mut File) -> Result<ReadStatus, String> {
    let mut buf = vec![0; READ_SIZE];
    match reader.read(buf.as_mut_slice()) {
        Ok(0) => return Ok(ReadStatus::EOF),
        Ok(n) => {
            buf.truncate(n);
            Ok(ReadStatus::Partial(buf))
        }
        Err(e) => Err(e.description().to_string()),
    }
}

fn read_all(reader: &mut File, progress_chan: &Sender<ProgressMsg>)
            -> Result<(), String> {
    loop {
        match try!(read_block(reader)) {
            ReadStatus::Partial(buf) => progress_chan.send(Payload(buf)).unwrap(),
            ReadStatus::EOF => return Ok(()),
        }
    }
}

pub fn factory(load_data: LoadData, classifier: Arc<MIMEClassifier>) {
    let url = load_data.url;
    let start_chan = load_data.consumer;
    assert!(&*url.scheme == "file");
    spawn_named("file_loader".to_owned(), move || {
        let metadata = Metadata::default(url.clone());
        let file_path: Result<PathBuf, ()> = url.to_file_path();
        match file_path {
            Ok(file_path) => {
                match File::open(&file_path) {
                    Ok(ref mut reader) => {
                        let res = read_block(reader);
                        let (res, progress_chan) = match res {
                            Ok(ReadStatus::Partial(buf)) => {
                                let progress_chan = start_sending_sniffed(start_chan, metadata,
                                                                          classifier, &buf);
{'fraction_non_alphanumeric': 0.14018480955600632, 'fraction_numerical': 0.006085192697768763, 'mean_word_length': 2.4376452362509684, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4125101', 'n_tokens_mistral': 1383, 'n_tokens_neox': 1352, 'n_words': 308}