<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use cookie_storage::CookieSource;
use resource_task::{Metadata, TargetedLoadResponse, LoadData, start_sending_opt, ResponseSenders};
use resource_task::ControlMsg;
use resource_task::ProgressMsg::{Payload, Done};

use log;
use std::collections::HashSet;
use file_loader;
use hyper::client::Request;
use hyper::header::common::{ContentLength, ContentType, Host, Location, SetCookie};
use hyper::HttpError;
use hyper::method::Method;
use hyper::net::HttpConnector;
use hyper::status::{StatusCode, StatusClass};
use std::error::Error;
use openssl::ssl::{SslContext, SslVerifyMode};
use std::io::{IoError, IoErrorKind, Reader};
use std::sync::mpsc::{Sender, channel};
use std::thunk::Invoke;
use util::task::spawn_named;
use util::resource_files::resources_dir_path;
use url::{Url, UrlParser};

use std::borrow::ToOwned;

pub fn factory(cookies_chan: Sender<ControlMsg>)
               -> Box<Invoke<(LoadData, Sender<TargetedLoadResponse>)> + Send> {
    box move |:(load_data, start_chan)| {
        spawn_named("http_loader".to_owned(), move || load(load_data, start_chan, cookies_chan))
    }
}

fn send_error(url: Url, err: String, senders: ResponseSenders) {
    let mut metadata = Metadata::default(url);
    metadata.status = None;

    match start_sending_opt(senders, metadata) {
        Ok(p) => p.send(Done(Err(err))).unwrap(),
        _ => {}
    };
}

fn load(mut load_data: LoadData, start_chan: Sender<TargetedLoadResponse>, cookies_chan: Sender<ControlMsg>) {
    // FIXME: At the time of writing this FIXME, servo didn't have any central
    //        location for configuration. If you're reading this and such a
    //        repository DOES exist, please update this constant to use it.
    let max_redirects = 50u;
    let mut iters = 0u;
    let mut url = load_data.url.clone();
    let mut redirected_to = HashSet::new();

    let senders = ResponseSenders {
        immediate_consumer: start_chan,
        eventual_consumer: load_data.consumer
    };

    // Loop to handle redirects.
    loop {
        iters = iters + 1;

        if iters > max_redirects {
            send_error(url, "too many redirects".to_string(), senders);
            return;
        }

        match url.scheme.as_slice() {
            "http" | "https" => {}
            _ => {
                let s = format!("{} request, but we don't support that scheme", url.scheme);
                send_error(url, s, senders);
                return;
            }
        }

        info!("requesting {}", url.serialize());

        fn verifier(ssl: &mut SslContext) {
            ssl.set_verify(SslVerifyMode::SslVerifyPeer, None);
            let mut certs = resources_dir_path();
            certs.push("certs");
            ssl.set_CA_file(&certs);
        };

        let ssl_err_string = "[UnknownError { library: \"SSL routines\", \
function: \"SSL3_GET_SERVER_CERTIFICATE\", \
reason: \"certificate verify failed\" }]";

        let mut connector = HttpConnector(Some(box verifier as Box<FnMut(&mut SslContext)>));
        let mut req = match Request::with_connector(load_data.method.clone(), url.clone(), &mut connector) {
            Ok(req) => req,
            Err(HttpError::HttpIoError(IoError {kind: IoErrorKind::OtherIoError,
                                                desc: "Error in OpenSSL",
                                                detail: Some(ref det)})) if det.as_slice() == ssl_err_string => {
                let mut image = resources_dir_path();
                image.push("badcert.html");
                let load_data = LoadData::new(Url::from_file_path(&image).unwrap(), senders.eventual_consumer);
                file_loader::factory(load_data, senders.immediate_consumer);
                return;
            },
            Err(e) => {
                println!("{:?}", e);
                send_error(url, e.description().to_string(), senders);
                return;
            }
        };

        let (tx, rx) = channel();
        cookies_chan.send(ControlMsg::GetCookiesForUrl(url.clone(), tx, CookieSource::HTTP));
        if let Some(cookies) = rx.recv().unwrap() {
            let mut v = Vec::new();
            v.push(cookies.into_bytes());
            load_data.headers.set_raw("Cookie".to_owned(), v);
        }

        // Avoid automatically preserving request headers when redirects occur.
        // See https://bugzilla.mozilla.org/show_bug.cgi?id=401564 and
        // https://bugzilla.mozilla.org/show_bug.cgi?id=216828
        if iters == 1 {
            // Preserve the `host` header set automatically by Request.
            let host = req.headers().get::<Host>().unwrap().clone();
            *req.headers_mut() = load_data.headers.clone();
            req.headers_mut().set(host);
        }

        // FIXME(seanmonstar): use AcceptEncoding from Hyper once available
        //if !req.headers.has::<AcceptEncoding>() {
            // We currently don't support HTTP Compression (FIXME #2587)
            req.headers_mut().set_raw("Accept-Encoding".to_owned(), vec![b"identity".to_vec()]);
        //}
        if log_enabled!(log::INFO) {
            info!("{}", load_data.method);
            for header in req.headers().iter() {
                info!(" - {}", header);
            }
            info!("{:?}", load_data.data);
        }

        // Avoid automatically sending request body if a redirect has occurred.
        let writer = match load_data.data {
            Some(ref data) if iters == 1 => {
                req.headers_mut().set(ContentLength(data.len() as u64));
                let mut writer = match req.start() {<commit_msg>Set cookie headers on the request, rather than the load data, to ensure they are used after redirects occur.<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use cookie_storage::CookieSource;
use resource_task::{Metadata, TargetedLoadResponse, LoadData, start_sending_opt, ResponseSenders};
use resource_task::ControlMsg;
use resource_task::ProgressMsg::{Payload, Done};

use log;
use std::collections::HashSet;
use file_loader;
use hyper::client::Request;
use hyper::header::common::{ContentLength, ContentType, Host, Location, SetCookie};
use hyper::HttpError;
use hyper::method::Method;
use hyper::net::HttpConnector;
use hyper::status::{StatusCode, StatusClass};
use std::error::Error;
use openssl::ssl::{SslContext, SslVerifyMode};
use std::io::{IoError, IoErrorKind, Reader};
use std::sync::mpsc::{Sender, channel};
use std::thunk::Invoke;
use util::task::spawn_named;
use util::resource_files::resources_dir_path;
use url::{Url, UrlParser};

use std::borrow::ToOwned;

pub fn factory(cookies_chan: Sender<ControlMsg>)
               -> Box<Invoke<(LoadData, Sender<TargetedLoadResponse>)> + Send> {
    box move |:(load_data, start_chan)| {
        spawn_named("http_loader".to_owned(), move || load(load_data, start_chan, cookies_chan))
    }
}

fn send_error(url: Url, err: String, senders: ResponseSenders) {
    let mut metadata = Metadata::default(url);
    metadata.status = None;

    match start_sending_opt(senders, metadata) {
        Ok(p) => p.send(Done(Err(err))).unwrap(),
        _ => {}
    };
}

fn load(mut load_data: LoadData, start_chan: Sender<TargetedLoadResponse>, cookies_chan: Sender<ControlMsg>) {
    // FIXME: At the time of writing this FIXME, servo didn't have any central
    //        location for configuration. If you're reading this and such a
    //        repository DOES exist, please update this constant to use it.
    let max_redirects = 50u;
    let mut iters = 0u;
    let mut url = load_data.url.clone();
    let mut redirected_to = HashSet::new();

    let senders = ResponseSenders {
        immediate_consumer: start_chan,
        eventual_consumer: load_data.consumer
    };

    // Loop to handle redirects.
    loop {
        iters = iters + 1;

        if iters > max_redirects {
            send_error(url, "too many redirects".to_string(), senders);
            return;
        }

        match url.scheme.as_slice() {
            "http" | "https" => {}
            _ => {
                let s = format!("{} request, but we don't support that scheme", url.scheme);
                send_error(url, s, senders);
                return;
            }
        }

        info!("requesting {}", url.serialize());

        fn verifier(ssl: &mut SslContext) {
            ssl.set_verify(SslVerifyMode::SslVerifyPeer, None);
            let mut certs = resources_dir_path();
            certs.push("certs");
            ssl.set_CA_file(&certs);
        };

        let ssl_err_string = "[UnknownError { library: \"SSL routines\", \
function: \"SSL3_GET_SERVER_CERTIFICATE\", \
reason: \"certificate verify failed\" }]";

        let mut connector = HttpConnector(Some(box verifier as Box<FnMut(&mut SslContext)>));
        let mut req = match Request::with_connector(load_data.method.clone(), url.clone(), &mut connector) {
            Ok(req) => req,
            Err(HttpError::HttpIoError(IoError {kind: IoErrorKind::OtherIoError,
                                                desc: "Error in OpenSSL",
                                                detail: Some(ref det)})) if det.as_slice() == ssl_err_string => {
                let mut image = resources_dir_path();
                image.push("badcert.html");
                let load_data = LoadData::new(Url::from_file_path(&image).unwrap(), senders.eventual_consumer);
                file_loader::factory(load_data, senders.immediate_consumer);
                return;
            },
            Err(e) => {
                println!("{:?}", e);
                send_error(url, e.description().to_string(), senders);
                return;
            }
        };

        // Avoid automatically preserving request headers when redirects occur.
        // See https://bugzilla.mozilla.org/show_bug.cgi?id=401564 and
        // https://bugzilla.mozilla.org/show_bug.cgi?id=216828
        if iters == 1 {
            // Preserve the `host` header set automatically by Request.
            let host = req.headers().get::<Host>().unwrap().clone();
            *req.headers_mut() = load_data.headers.clone();
            req.headers_mut().set(host);
        }

        let (tx, rx) = channel();
        cookies_chan.send(ControlMsg::GetCookiesForUrl(url.clone(), tx, CookieSource::HTTP));
        if let Some(cookies) = rx.recv().unwrap() {
            let mut v = Vec::new();
            v.push(cookies.into_bytes());
            load_data.headers.set_raw("Cookie".to_owned(), v);
        }

        // FIXME(seanmonstar): use AcceptEncoding from Hyper once available
        //if !req.headers.has::<AcceptEncoding>() {
            // We currently don't support HTTP Compression (FIXME #2587)
            req.headers_mut().set_raw("Accept-Encoding".to_owned(), vec![b"identity".to_vec()]);
        //}
        if log_enabled!(log::INFO) {
            info!("{}", load_data.method);
            for header in req.headers().iter() {
                info!(" - {}", header);
            }
            info!("{:?}", load_data.data);
        }

        // Avoid automatically sending request body if a redirect has occurred.
        let writer = match load_data.data {
            Some(ref data) if iters == 1 => {
                req.headers_mut().set(ContentLength(data.len() as u64));
                let mut writer = match req.start() {
{'fraction_non_alphanumeric': 0.13450890195076243, 'fraction_numerical': 0.004940795638470057, 'mean_word_length': 2.483679525222552, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 41, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1979849', 'n_tokens_mistral': 3486, 'n_tokens_neox': 3395, 'n_words': 834}