<commit_before>

use errors::*;
use notifications::Notification;
use sha2::Sha256;
use std::env;
use std::path::Path;
use url::Url;

pub fn download_file(url: &Url,
                     path: &Path,
                     hasher: Option<&mut Sha256>,
                     notify_handler: &Fn(Notification))
                     -> Result<()> {
    if env::var_os("RUSTUP_USE_HYPER").is_some() {
        self::hyper::download_file(url, path, hasher, notify_handler)
    } else {
        self::curl::download_file(url, path, hasher, notify_handler)
    }
}

mod curl {
    use curl::easy::Easy;
    use errors::*;
    use notifications::Notification;
    use sha2::{Sha256, Digest};
    use std::cell::RefCell;
    use std::fs;
    use std::path::Path;
    use std::str;
    use std::time::Duration;
    use url::Url;

    pub fn download_file(url: &Url,
                         path: &Path,
                         mut hasher: Option<&mut Sha256>,
                         notify_handler: &Fn(Notification))
                         -> Result<()> {
        use notifications::Notification;
        use std::io::Write;

        let mut file = try!(fs::File::create(&path).chain_err(
            || "error creating file for download"));

        // Fetch either a cached libcurl handle (which will preserve open
        // connections) or create a new one if it isn't listed.
        //
        // Once we've acquired it, reset the lifetime from 'static to our local
        // scope.
        thread_local!(static EASY: RefCell<Easy> = RefCell::new(Easy::new()));
        EASY.with(|handle| {
            let mut handle = handle.borrow_mut();

            try!(handle.url(&url.to_string()).chain_err(|| "failed to set url"));
            try!(handle.follow_location(true).chain_err(|| "failed to set follow redirects"));

            // Take at most 30s to connect
            try!(handle.connect_timeout(Duration::new(30, 0)).chain_err(|| "failed to set connect timeout"));

            // Fail if less than 10 bytes are transferred every 30 seconds
            try!(handle.low_speed_limit(10).chain_err(|| "failed to set low speed limit"));
            try!(handle.low_speed_time(Duration::new(30, 0)).chain_err(|| "failed to set low speed time"));

            {
                let fserr = RefCell::new(None);
                let mut transfer = handle.transfer();

                // Data callback for libcurl which is called with data that's
                // downloaded. We just feed it into our hasher and also write it out
                // to disk.
                try!(transfer.write_function(|data| {
                    if let Some(ref mut h) = hasher {
                        h.input(data);
                    }
                    notify_handler(Notification::DownloadDataReceived(data.len()));
                    match file.write_all(data) {
                        Ok(()) => Ok(data.len()),
                        Err(e) => {
                            *fserr.borrow_mut() = Some(e);
                            Ok(0)
                        }
                    }
                }).chain_err(|| "failed to set write"));

                // Listen for headers and parse out a `Content-Length` if it comes
                // so we know how much we're downloading.
                try!(transfer.header_function(|header| {
                    if let Ok(data) = str::from_utf8(header) {
                        let prefix = "Content-Length: ";
                        if data.starts_with(prefix) {
                            if let Ok(s) = data[prefix.len()..].trim().parse() {
                                let msg = Notification::DownloadContentLengthReceived(s);
                                notify_handler(msg);
                            }
                        }
                    }
                    true
                }).chain_err(|| "failed to set header"));

                // If an error happens check to see if we had a filesystem error up
                // in `fserr`, but we always want to punt it up.
                try!(transfer.perform().or_else(|e| {
                    match fserr.borrow_mut().take() {
                        Some(fs) => Err(fs).chain_err(|| ErrorKind::HttpError(e)),
                        None => Err(ErrorKind::HttpError(e).into())
                    }
                }));
            }

            // If we didn't get a 200 or 0 ("OK" for files) then return an error
            let code = try!(handle.response_code().chain_err(|| "failed to get response code"));
            if code != 200 && code != 0 {
                return Err(ErrorKind::HttpStatus(code).into());
            }

            notify_handler(Notification::DownloadFinished);
            Ok(())
        })
    }
}
<commit_msg>Add cursory docs to rust_utils::download<commit_after>//! Easy file downloading

use errors::*;
use notifications::Notification;
use sha2::Sha256;
use std::env;
use std::path::Path;
use url::Url;

pub fn download_file(url: &Url,
                     path: &Path,
                     hasher: Option<&mut Sha256>,
                     notify_handler: &Fn(Notification))
                     -> Result<()> {
    if env::var_os("RUSTUP_USE_HYPER").is_some() {
        self::hyper::download_file(url, path, hasher, notify_handler)
    } else {
        self::curl::download_file(url, path, hasher, notify_handler)
    }
}


/// Download via libcurl; encrypt with the native (or OpenSSl) TLS
/// stack via libcurl
mod curl {
    use curl::easy::Easy;
    use errors::*;
    use notifications::Notification;
    use sha2::{Sha256, Digest};
    use std::cell::RefCell;
    use std::fs;
    use std::path::Path;
    use std::str;
    use std::time::Duration;
    use url::Url;

    pub fn download_file(url: &Url,
                         path: &Path,
                         mut hasher: Option<&mut Sha256>,
                         notify_handler: &Fn(Notification))
                         -> Result<()> {
        use notifications::Notification;
        use std::io::Write;

        let mut file = try!(fs::File::create(&path).chain_err(
            || "error creating file for download"));

        // Fetch either a cached libcurl handle (which will preserve open
        // connections) or create a new one if it isn't listed.
        //
        // Once we've acquired it, reset the lifetime from 'static to our local
        // scope.
        thread_local!(static EASY: RefCell<Easy> = RefCell::new(Easy::new()));
        EASY.with(|handle| {
            let mut handle = handle.borrow_mut();

            try!(handle.url(&url.to_string()).chain_err(|| "failed to set url"));
            try!(handle.follow_location(true).chain_err(|| "failed to set follow redirects"));

            // Take at most 30s to connect
            try!(handle.connect_timeout(Duration::new(30, 0)).chain_err(|| "failed to set connect timeout"));

            // Fail if less than 10 bytes are transferred every 30 seconds
            try!(handle.low_speed_limit(10).chain_err(|| "failed to set low speed limit"));
            try!(handle.low_speed_time(Duration::new(30, 0)).chain_err(|| "failed to set low speed time"));

            {
                let fserr = RefCell::new(None);
                let mut transfer = handle.transfer();

                // Data callback for libcurl which is called with data that's
                // downloaded. We just feed it into our hasher and also write it out
                // to disk.
                try!(transfer.write_function(|data| {
                    if let Some(ref mut h) = hasher {
                        h.input(data);
                    }
                    notify_handler(Notification::DownloadDataReceived(data.len()));
                    match file.write_all(data) {
                        Ok(()) => Ok(data.len()),
                        Err(e) => {
                            *fserr.borrow_mut() = Some(e);
                            Ok(0)
                        }
                    }
                }).chain_err(|| "failed to set write"));

                // Listen for headers and parse out a `Content-Length` if it comes
                // so we know how much we're downloading.
                try!(transfer.header_function(|header| {
                    if let Ok(data) = str::from_utf8(header) {
                        let prefix = "Content-Length: ";
                        if data.starts_with(prefix) {
                            if let Ok(s) = data[prefix.len()..].trim().parse() {
                                let msg = Notification::DownloadContentLengthReceived(s);
                                notify_handler(msg);
                            }
                        }
                    }
                    true
                }).chain_err(|| "failed to set header"));

                // If an error happens check to see if we had a filesystem error up
                // in `fserr`, but we always want to punt it up.
                try!(transfer.perform().or_else(|e| {
                    match fserr.borrow_mut().take() {
                        Some(fs) => Err(fs).chain_err(|| ErrorKind::HttpError(e)),
                        None => Err(ErrorKind::HttpError(e).into())
                    }
                }));
            }

            // If we didn't get a 200 or 0 ("OK" for files) then return an error
            let code = try!(handle.response_code().chain_err(|| "failed to get response code"));
            if code != 200 && code != 0 {
                return Err(ErrorKind::HttpStatus(code).into());
            }

            notify_handler(Notification::DownloadFinished);
            Ok(())
        })
    }
}

/// Download via hyper; encrypt with the native (or OpenSSl) TLS
/// stack via native-tls
{'fraction_non_alphanumeric': 0.1213979153893317, 'fraction_numerical': 0.007766196607398324, 'mean_word_length': 1.534179181771103, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5900307', 'n_tokens_mistral': 2693, 'n_tokens_neox': 2473, 'n_words': 707}