<commit_before>//! Contains all HTML-related code. See the `Page` documentation for details.

// TODO: Why do I need these two lines "extern crate html5ever" and "extern crate
// html5ever_dom_sink" both here and in main.rs?
extern crate html5ever;
extern crate html5ever_dom_sink;
extern crate rand;
extern crate tendril;

use std::str::FromStr;
use std::sync::mpsc::{channel, Receiver, Sender};
use std::thread;

use html5ever::Attribute;
use html5ever::tree_builder::interface::TreeSink;
use html5ever_dom_sink::common::NodeEnum;
use html5ever_dom_sink::rcdom::Handle;
use html5ever_dom_sink::rcdom::RcDom;
use regex::Captures;

use wiki::Wiki;

// TODO: massive cleanup, all over this file.
// TODO: I'm using "body" and even "html_body" to mean "article body" in this file and main.rs,
// which isn't right. Come up with a better name for that and use it everywhere.

/// Represents, and owns all behavior related to, the contents of the HTML page shown to the
/// user. This includes fetching the rendered article from Wikipedia, replacing its contents with
/// the rendered wikitext, and processing/removing merge markers.
///
/// The API to this struct comprises two methods: Page::new() constructs a new Page. It should be
/// called early, so the Page can start fetching article HTML from Wikipedia.
/// Page:replace_body_and_remove_merge_markers() processes the merge markers in the rendered
/// wikitext and puts the header and footer around it.
pub struct Page {
    html_body_sender: Sender<String>,
    replaced_body_receiver: Receiver<Result<String, String>>,
}

impl Page {
    /// Creates a new Page representing the article at `title`. This kicks off a background thread
    /// that fetches the current article HTML from Wikipedia. Because of that, it should be called
    /// as early as possible (as soon as the title being served is known), so that the page fetch
    /// stays off the critical path for page load.
    pub fn new(title: &str, wiki: Wiki) -> Page {
        let placeholder = format!("WMW_PLACEHOLDER_{}", rand::random::<u64>());
        let current_content_receiver =
            Page::spawn_content_fetch_thread(title, placeholder.clone(), wiki);
        let (html_body_sender, replaced_body_receiver) =
            Page::spawn_replace_body_thread(current_content_receiver, placeholder);
        Page {
            html_body_sender: html_body_sender,
            replaced_body_receiver: replaced_body_receiver,
        }
    }

    // TODO: Why is this done in a separate thread at all? That's unnecessary. Just do the
    // processing on this thread.
    /// This finishes the HTML processing - it replaces the merge markers in `html_body` with HTML
    /// tags, and inserts the resulting HTML into the page skeleton.
    pub fn replace_body_and_remove_merge_markers(&self, html_body: String)
                                                 -> Result<String, String> {
        self.html_body_sender.send(html_body);
        try_display!(self.replaced_body_receiver.recv(), "Failed to get data from channel")
    }

    fn spawn_content_fetch_thread(title: &str, placeholder: String, wiki: Wiki)
                                  -> Receiver<Result<String, String>> {
        let (sender, receiver) = channel::<Result<String, String>>();
        let title = title.to_owned().clone();
        thread::spawn(move|| {
            sender.send(
                match wiki.get_current_page_content(&title) {
                    Ok(content) =>
                        replace_node_with_placeholder(&content, "mw-content-text", &placeholder),
                Err(msg) => Err(msg),
            }).unwrap();
        });
        receiver
    }

    fn spawn_replace_body_thread(
        page_skeleton_receiver: Receiver<Result<String, String>>, placeholder: String)
        -> (Sender<String>, Receiver<Result<String, String>>) {
        let (html_body_sender, html_body_receiver) = channel::<String>();
        let (replaced_body_sender, replaced_body_receiver) = channel::<Result<String, String>>();
        thread::spawn(move|| {
            match (page_skeleton_receiver.recv(), html_body_receiver.recv()) {
                (Ok(Ok(page_skeleton)), Ok(body)) => {
                    // TODO: This should remove the merge markers first, and replace the page
                    // contents second. There are never merge markers in the page skeleton, so it's
                    // useless to look for merge markers there.
                    let html_with_merge_markers =
                        remove_merge_markers_from_html(page_skeleton.replace(&placeholder, &body));
                    // TODO: Move this elsewhere, use constants, etc.
                    let start_regex = regex!("\u{E000}([0-9]+)\u{E000}");
                    let end_regex = regex!("\u{E001}[0-9]+\u{E001}");
                    let finished_html =
                        start_regex.replace_all(
                            &end_regex.replace_all(&html_with_merge_markers, "</span>"),
                            |captures: &Captures|
                            format!("<span style=\"color: red\" class=\"vandalism-{}\">",
                                    captures.at(1).unwrap()));
                    replaced_body_sender.send(Ok(finished_html))
                },
                (Ok(Err(msg)), _) => replaced_body_sender.send(Err(msg)),
                (Err(err), _) => replaced_body_sender.send(Err(format!("error: {}", err))),
                (_, Err(err)) => replaced_body_sender.send(Err(format!("error: {}", err))),
            }.unwrap()
        });
        (html_body_sender, replaced_body_receiver)
    }
}

fn replace_node_with_placeholder(original_html: &str, div_id: &str, placeholder: &str)
    -> Result<String, String> {
    let html = tendril::StrTendril::from_str(original_html).unwrap();
    let mut dom: RcDom = html5ever::parse(html5ever::one_input(html), Default::default());

    let handle = try!(find_node_by_id(&dom.get_document(), div_id));
    let child_handles =
        (&handle.borrow().children).into_iter().map(|child| child.clone()).collect::<Vec<_>>();
    for child_handle in child_handles {
        dom.remove_from_parent(child_handle);
    }
    dom.append(handle,
               html5ever::tree_builder::interface::NodeOrText::AppendText(
                   tendril::StrTendril::from_str(placeholder).unwrap()));
    let mut serialized: Vec<u8> = vec![];
    try_display!(
        html5ever::serialize::serialize(&mut serialized, &dom.document, Default::default()),
        "Failed to serialize modified HTML");<commit_msg>Remove the useless HTML-processing thread, which only does processing on the critical path anyway.<commit_after>//! Contains all HTML-related code. See the `Page` documentation for details.

// TODO: Why do I need these two lines "extern crate html5ever" and "extern crate
// html5ever_dom_sink" both here and in main.rs?
extern crate html5ever;
extern crate html5ever_dom_sink;
extern crate rand;
extern crate tendril;

use std::str::FromStr;
use std::sync::mpsc::{channel, Receiver, Sender};
use std::thread;

use html5ever::Attribute;
use html5ever::tree_builder::interface::TreeSink;
use html5ever_dom_sink::common::NodeEnum;
use html5ever_dom_sink::rcdom::Handle;
use html5ever_dom_sink::rcdom::RcDom;
use regex::Captures;

use wiki::Wiki;

// TODO: massive cleanup, all over this file.
// TODO: I'm using "body" and even "html_body" to mean "article body" in this file and main.rs,
// which isn't right. Come up with a better name for that and use it everywhere.

/// Represents, and owns all behavior related to, the contents of the HTML page shown to the
/// user. This includes fetching the rendered article from Wikipedia, replacing its contents with
/// the rendered wikitext, and processing/removing merge markers.
///
/// The API to this struct comprises two methods: Page::new() constructs a new Page. It should be
/// called early, so the Page can start fetching article HTML from Wikipedia.
/// Page:replace_body_and_remove_merge_markers() processes the merge markers in the rendered
/// wikitext and puts the header and footer around it.
pub struct Page {
    placeholder: String,
    page_skeleton_receiver: Receiver<Result<String, String>>,
}

impl Page {
    /// Creates a new Page representing the article at `title`. This kicks off a background thread
    /// that fetches the current article HTML from Wikipedia. Because of that, it should be called
    /// as early as possible (as soon as the title being served is known), so that the page fetch
    /// stays off the critical path for page load.
    pub fn new(title: &str, wiki: Wiki) -> Page {
        let placeholder = format!("WMW_PLACEHOLDER_{}", rand::random::<u64>());
        let page_skeleton_receiver =
            Page::spawn_page_skeleton_fetch_thread(title, placeholder.clone(), wiki);
        Page {
            placeholder: placeholder,
            page_skeleton_receiver: page_skeleton_receiver,
        }
    }

    /// This finishes the HTML processing - it replaces the merge markers in `html_body` with HTML
    /// tags, and inserts the resulting HTML into the page skeleton.
    pub fn replace_body_and_remove_merge_markers(&self, html_body: String)
                                                 -> Result<String, String> {
        match self.page_skeleton_receiver.recv() {
            Ok(Ok(page_skeleton)) => {
                // TODO: This should remove the merge markers first, and replace the page
                // contents second. There are never merge markers in the page skeleton, so it's
                // useless to look for merge markers there.
                let html_with_merge_markers =
                    remove_merge_markers_from_html(
                        page_skeleton.replace(&self.placeholder, &html_body));
                // TODO: Move this elsewhere, use constants, etc.
                let start_regex = regex!("\u{E000}([0-9]+)\u{E000}");
                let end_regex = regex!("\u{E001}[0-9]+\u{E001}");
                let finished_html =
                    start_regex.replace_all(
                        &end_regex.replace_all(&html_with_merge_markers, "</span>"),
                        |captures: &Captures|
                        format!("<span style=\"color: red\" class=\"vandalism-{}\">",
                                captures.at(1).unwrap()));
                Ok(finished_html)
            },
            Ok(Err(msg))=> Err(msg),
            Err(err) => Err(format!("error: {}", err)),
        }
    }

    fn spawn_page_skeleton_fetch_thread(title: &str, placeholder: String, wiki: Wiki)
                                  -> Receiver<Result<String, String>> {
        let (page_skeleton_sender, page_skeleton_receiver) = channel::<Result<String, String>>();
        let title = title.to_owned().clone();
        thread::spawn(move|| {
            page_skeleton_sender.send(
                match wiki.get_current_page_content(&title) {
                    Ok(content) =>
                        replace_node_with_placeholder(&content, "mw-content-text", &placeholder),
                    Err(msg) => Err(msg),
                }).unwrap();
        });
        page_skeleton_receiver
    }
}

fn replace_node_with_placeholder(original_html: &str, div_id: &str, placeholder: &str)
    -> Result<String, String> {
    let html = tendril::StrTendril::from_str(original_html).unwrap();
    let mut dom: RcDom = html5ever::parse(html5ever::one_input(html), Default::default());

    let handle = try!(find_node_by_id(&dom.get_document(), div_id));
    let child_handles =
        (&handle.borrow().children).into_iter().map(|child| child.clone()).collect::<Vec<_>>();
    for child_handle in child_handles {
        dom.remove_from_parent(child_handle);
    }
    dom.append(handle,
               html5ever::tree_builder::interface::NodeOrText::AppendText(
                   tendril::StrTendril::from_str(placeholder).unwrap()));
    let mut serialized: Vec<u8> = vec![];
    try_display!(
        html5ever::serialize::serialize(&mut serialized, &dom.document, Default::default()),
        "Failed to serialize modified HTML");
{'fraction_non_alphanumeric': 0.1152043417482115, 'fraction_numerical': 0.005427185264369707, 'mean_word_length': 2.7181290125343933, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 60, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4955831', 'n_tokens_mistral': 3464, 'n_tokens_neox': 3355, 'n_words': 974}