<commit_before>use html5ever::interface::QualName;
use html5ever::parse_document;
use html5ever::rcdom::{Handle, NodeData, RcDom};
use html5ever::serialize::{serialize, SerializeOpts};
use html5ever::tendril::{format_tendril, TendrilSink};
use html5ever::tree_builder::{Attribute, TreeSink};
use html5ever::{local_name, namespace_url, ns};
use http::retrieve_asset;
use js::attr_is_event_handler;
use std::collections::HashMap;
use std::default::Default;
use utils::{data_to_dataurl, is_valid_url, resolve_css_imports, resolve_url, url_has_protocol};

lazy_static! {
    static ref EMPTY_STRING: String = String::new();
}

const ICON_VALUES: [&str; 5] = [
    "icon",
    "shortcut icon",
    "mask-icon",
    "apple-touch-icon",
    "fluid-icon",
];

const TRANSPARENT_PIXEL: &str =
    "data:image/png;base64,\
     iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=";

pub fn get_parent_node(node: &Handle) -> Handle {
    let parent = node.parent.take().clone();
    parent.and_then(|node| node.upgrade()).unwrap()
}

pub fn get_node_name(node: &Handle) -> String {
    match &node.data {
        NodeData::Element { ref name, .. } => name.local.as_ref().to_string(),
        _ => EMPTY_STRING.clone(),
    }
}

pub fn is_icon(attr_value: &str) -> bool {
    ICON_VALUES.contains(&&*attr_value.to_lowercase())
}

pub fn walk_and_embed_assets(
    cache: &mut HashMap<String, String>,
    url: &str,
    node: &Handle,
    opt_no_css: bool,
    opt_no_js: bool,
    opt_no_images: bool,
    opt_user_agent: &str,
    opt_silent: bool,
    opt_insecure: bool,
    opt_no_frames: bool,
) {
    match node.data {
        NodeData::Document => {
            // Dig deeper
            for child in node.children.borrow().iter() {
                walk_and_embed_assets(
                    cache,
                    &url,
                    child,
                    opt_no_css,
                    opt_no_js,
                    opt_no_images,
                    opt_user_agent,
                    opt_silent,
                    opt_insecure,
                    opt_no_frames,
                );
            }
        }
        NodeData::Element {
            ref name,
            ref attrs,
            ..
        } => {
            let attrs_mut = &mut attrs.borrow_mut();

            match name.local.as_ref() {
                "link" => {
                    let mut link_type: &str = "";

                    for attr in attrs_mut.iter_mut() {
                        if &attr.name.local == "rel" {
                            if is_icon(&attr.value.to_string()) {
                                link_type = "icon";
                                break;
                            } else if attr.value.to_string() == "stylesheet" {
                                link_type = "stylesheet";
                                break;
                            }
                        }
                    }

                    if link_type == "icon" {
                        for attr in attrs_mut.iter_mut() {
                            if &attr.name.local == "href" {
                                if opt_no_images {
                                    attr.value.clear();
                                } else {
                                    let href_full_url: String =
                                        resolve_url(&url, &attr.value.to_string())
                                            .unwrap_or(EMPTY_STRING.clone());
                                    let (favicon_dataurl, _) = retrieve_asset(
                                        cache,
                                        &href_full_url,
                                        true,
                                        "",
                                        opt_user_agent,
                                        opt_silent,
                                        opt_insecure,
                                    )
                                    .unwrap_or((EMPTY_STRING.clone(), EMPTY_STRING.clone()));
                                    attr.value.clear();
                                    attr.value.push_slice(favicon_dataurl.as_str());
                                }
                            }
                        }
                    } else if link_type == "stylesheet" {
                        for attr in attrs_mut.iter_mut() {
                            if &attr.name.local == "href" {
                                if opt_no_css {
                                    attr.value.clear();
                                } else {
                                    let href_full_url: String =
                                        resolve_url(&url, &attr.value.to_string())
                                            .unwrap_or(EMPTY_STRING.clone());
                                    let (css, _) = retrieve_asset(
                                        cache,
                                        &href_full_url,
                                        false,
                                        "text/css",
                                        opt_user_agent,
                                        opt_silent,
                                        opt_insecure,
                                    )
                                    .unwrap_or((EMPTY_STRING.clone(), EMPTY_STRING.clone()));
                                    attr.value.clear();

                                    let css_resolved = resolve_css_imports(
                                        cache,
                                        &css,
                                        &href_full_url,
                                        opt_user_agent,
                                        opt_silent,
                                        opt_insecure,
                                    );

                                    attr.value.push_slice(css_resolved.as_str());
                                }
                            }
                        }
                    } else {
                        for attr in attrs_mut.iter_mut() {
                            if &attr.name.local == "href" {
                                let href_full_url: String =
                                    resolve_url(&url, &attr.value.to_string())
                                        .unwrap_or(EMPTY_STRING.clone());
                                attr.value.clear();
                                attr.value.push_slice(&href_full_url.as_str());
                            }
                        }
                    }<commit_msg>Add warning and fallback when parsing a rel=stylesheet link<commit_after>use html5ever::interface::QualName;
use html5ever::parse_document;
use html5ever::rcdom::{Handle, NodeData, RcDom};
use html5ever::serialize::{serialize, SerializeOpts};
use html5ever::tendril::{format_tendril, TendrilSink};
use html5ever::tree_builder::{Attribute, TreeSink};
use html5ever::{local_name, namespace_url, ns};
use http::retrieve_asset;
use js::attr_is_event_handler;
use std::io::{stderr, Write};
use std::collections::HashMap;
use std::default::Default;
use utils::{data_to_dataurl, is_valid_url, resolve_css_imports, resolve_url, url_has_protocol};

lazy_static! {
    static ref EMPTY_STRING: String = String::new();
}

const ICON_VALUES: [&str; 5] = [
    "icon",
    "shortcut icon",
    "mask-icon",
    "apple-touch-icon",
    "fluid-icon",
];

const TRANSPARENT_PIXEL: &str =
    "data:image/png;base64,\
     iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=";

pub fn get_parent_node(node: &Handle) -> Handle {
    let parent = node.parent.take().clone();
    parent.and_then(|node| node.upgrade()).unwrap()
}

pub fn get_node_name(node: &Handle) -> String {
    match &node.data {
        NodeData::Element { ref name, .. } => name.local.as_ref().to_string(),
        _ => EMPTY_STRING.clone(),
    }
}

pub fn is_icon(attr_value: &str) -> bool {
    ICON_VALUES.contains(&&*attr_value.to_lowercase())
}

pub fn walk_and_embed_assets(
    cache: &mut HashMap<String, String>,
    url: &str,
    node: &Handle,
    opt_no_css: bool,
    opt_no_js: bool,
    opt_no_images: bool,
    opt_user_agent: &str,
    opt_silent: bool,
    opt_insecure: bool,
    opt_no_frames: bool,
) {
    match node.data {
        NodeData::Document => {
            // Dig deeper
            for child in node.children.borrow().iter() {
                walk_and_embed_assets(
                    cache,
                    &url,
                    child,
                    opt_no_css,
                    opt_no_js,
                    opt_no_images,
                    opt_user_agent,
                    opt_silent,
                    opt_insecure,
                    opt_no_frames,
                );
            }
        }
        NodeData::Element {
            ref name,
            ref attrs,
            ..
        } => {
            let attrs_mut = &mut attrs.borrow_mut();

            match name.local.as_ref() {
                "link" => {
                    let mut link_type: &str = "";

                    for attr in attrs_mut.iter_mut() {
                        if &attr.name.local == "rel" {
                            if is_icon(&attr.value.to_string()) {
                                link_type = "icon";
                                break;
                            } else if attr.value.to_string() == "stylesheet" {
                                link_type = "stylesheet";
                                break;
                            }
                        }
                    }

                    if link_type == "icon" {
                        for attr in attrs_mut.iter_mut() {
                            if &attr.name.local == "href" {
                                if opt_no_images {
                                    attr.value.clear();
                                } else {
                                    let href_full_url: String =
                                        resolve_url(&url, &attr.value.to_string())
                                            .unwrap_or(EMPTY_STRING.clone());
                                    let (favicon_dataurl, _) = retrieve_asset(
                                        cache,
                                        &href_full_url,
                                        true,
                                        "",
                                        opt_user_agent,
                                        opt_silent,
                                        opt_insecure,
                                    )
                                    .unwrap_or((EMPTY_STRING.clone(), EMPTY_STRING.clone()));
                                    attr.value.clear();
                                    attr.value.push_slice(favicon_dataurl.as_str());
                                }
                            }
                        }
                    } else if link_type == "stylesheet" {
                        for attr in attrs_mut.iter_mut() {
                            if &attr.name.local == "href" {
                                if opt_no_css {
                                    attr.value.clear();
                                } else {
                                    let href_full_url: String =
                                        resolve_url(&url, &attr.value.to_string())
                                            .unwrap_or(EMPTY_STRING.clone());
                                    let replacement_text = match retrieve_asset(
                                        cache,
                                        &href_full_url,
                                        false,
                                        "text/css",
                                        opt_user_agent,
                                        opt_silent,
                                        opt_insecure,
                                    ) {

                                        // On successful retrieval, traverse CSS
                                        Ok((css_data, _)) => resolve_css_imports(
                                            cache,
                                            &css_data,
                                            &href_full_url,
                                            opt_user_agent,
                                            opt_silent,
                                            opt_insecure,
                                        ),

                                        // If a network error occured, warn
                                        Err(e) => {
                                            writeln!(
                                                stderr(),
                                                "Warning: {}",
                                                e,
                                            ).unwrap();

                                            //If failed to resolve, replace with absolute URL
                                            href_full_url
                                        },
                                    };

                                    attr.value.clear();
                                    attr.value.push_slice(&replacement_text);
                                }
                            }
                        }
                    } else {
                        for attr in attrs_mut.iter_mut() {
                            if &attr.name.local == "href" {
                                let href_full_url: String =
                                    resolve_url(&url, &attr.value.to_string())
                                        .unwrap_or(EMPTY_STRING.clone());
                                attr.value.clear();
                                attr.value.push_slice(&href_full_url.as_str());
                            }
                        }
                    }
{'fraction_non_alphanumeric': 0.08545609790806888, 'fraction_numerical': 0.00256154831364736, 'mean_word_length': 0.8265107212475633, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 124, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1294153', 'n_tokens_mistral': 3282, 'n_tokens_neox': 3184, 'n_words': 549}