<commit_before>//! Provides sourcemap validation functionality.
use std::fs;
use std::fmt;
use std::io::Read;
use std::cell::RefCell;
use std::collections::{HashSet, HashMap};
use std::iter::FromIterator;
use std::path::{Path, PathBuf};

use url::Url;
use api::{Api, FileContents};

use sourcemap;
use might_be_minified;
use indicatif::{ProgressBar, ProgressStyle, ProgressDrawTarget, Term, style};

use prelude::*;

fn make_progress_bar(len: u64) -> ProgressBar {
    let pb = ProgressBar::new(len);
    pb.set_draw_target(ProgressDrawTarget::to_term(Term::stdout(), None));
    pb.set_style(ProgressStyle::default_bar()
        .template(&format!("{} {{msg}}\n{{wide_bar}} {{pos}}/{{len}}",
                           style(">").cyan())));
    pb
}

fn join_url(base_url: &str, url: &str) -> Result<String> {
    if base_url.starts_with("~/") {
        match Url::parse(&format!("http://{}", base_url))?.join(url) {
            Ok(url) => {
                let rv = url.to_string();
                if rv.starts_with("http://~/") {
                    Ok(format!("~/{}", &rv[9..]))
                } else {
                    Ok(rv)
                }
            }
            Err(x) => fail!(x),
        }
    } else {
        Ok(Url::parse(base_url)?.join(url)?.to_string())
    }
}

fn split_url(url: &str) -> (Option<&str>, &str, Option<&str>) {
    let mut part_iter = url.rsplitn(2, '/');
    let (filename, ext) = part_iter.next()
        .map(|x| {
            let mut fn_iter = x.splitn(2, '.');
            (fn_iter.next(), fn_iter.next())
        })
        .unwrap_or((None, None));
    let path = part_iter.next();
    (path, filename.unwrap_or(""), ext)
}

fn unsplit_url(path: Option<&str>, basename: &str, ext: Option<&str>) -> String {
    let mut rv = String::new();
    if let Some(path) = path {
        rv.push_str(path);
        rv.push('/');
    }
    rv.push_str(basename);
    if let Some(ext) = ext {
        rv.push('.');
        rv.push_str(ext);
    }
    rv
}

pub fn get_sourcemap_reference_from_headers<'a, I: Iterator<Item = (&'a String, &'a String)>>
    (headers: I)
     -> Option<&'a str> {
    for (k, v) in headers {
        let ki = &k.to_lowercase();
        if ki == "sourcemap" || ki == "x-sourcemap" {
            return Some(v.as_str());
        }
    }
    None
}


fn find_sourcemap_reference(sourcemaps: &HashSet<String>, min_url: &str) -> Result<String> {
    // if there is only one sourcemap in total we just assume that's the one.
    // We just need to make sure that we fix up the reference if we need to
    // (eg: ~/ -> /).
    if sourcemaps.len() == 1 {
        return Ok(sourcemap::make_relative_path(
            min_url, sourcemaps.iter().next().unwrap()));
    }

    let map_ext = "map";
    let (path, basename, ext) = split_url(min_url);

    // foo.min.js -> foo.map
    if sourcemaps.contains(&unsplit_url(path, basename, Some("map"))) {
        return Ok(unsplit_url(None, basename, Some("map")));
    }

    if let Some(ext) = ext.as_ref() {
        // foo.min.js -> foo.min.js.map
        let new_ext = format!("{}.{}", ext, map_ext);
        if sourcemaps.contains(&unsplit_url(path, basename, Some(&new_ext))) {
            return Ok(unsplit_url(None, basename, Some(&new_ext)));
        }

        // foo.min.js -> foo.js.map
        if ext.starts_with("min.") {
            let new_ext = format!("{}.{}", &ext[4..], map_ext);
            if sourcemaps.contains(&unsplit_url(path, basename, Some(&new_ext))) {
                return Ok(unsplit_url(None, basename, Some(&new_ext)));
            }
        }

        // foo.min.js -> foo.min.map
        let mut parts: Vec<_> = ext.split('.').collect();
        if parts.len() > 1 {
            let parts_len = parts.len();
            parts[parts_len - 1] = &map_ext;
            let new_ext = parts.join(".");
            if sourcemaps.contains(&unsplit_url(path, basename, Some(&new_ext))) {
                return Ok(unsplit_url(None, basename, Some(&new_ext)));
            }
        }
    }

    fail!("Could not auto-detect referenced sourcemap for {}.",
          min_url);
}


#[derive(PartialEq, Eq, PartialOrd, Ord, Copy, Clone)]
enum SourceType {
    Script,
    MinifiedScript,
    SourceMap,
}


#[derive(PartialEq, Debug)]
enum LogLevel {
    Warning,
    Error,
}

impl fmt::Display for LogLevel {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            LogLevel::Warning => write!(f, "warning"),
            LogLevel::Error => write!(f, "error"),
        }
    }
}

struct Source {
    url: String,
    #[allow(unused)]
    file_path: PathBuf,
    contents: String,
    ty: SourceType,
    skip_upload: bool,
    headers: Vec<(String, String)>,
    messages: RefCell<Vec<(LogLevel, String)>>,
}

pub struct SourceMapProcessor {
    pending_sources: HashSet<(String, PathBuf)>,
    sources: HashMap<String, Source>,
}

impl Source {

    fn log(&self, level: LogLevel, msg: String) {
        self.messages.borrow_mut().push((level, msg));
    }

    fn warn(&self, msg: String) {
        self.log(LogLevel::Warning, msg);
    }

    fn error(&self, msg: String) {
        self.log(LogLevel::Error, msg);
    }
}

impl SourceMapProcessor {
    /// Creates a new sourcemap validator.
    pub fn new() -> SourceMapProcessor {
        SourceMapProcessor {
            pending_sources: HashSet::new(),
            sources: HashMap::new(),
        }
    }

    /// Adds a new file for processing.
    pub fn add(&mut self, url: &str, path: &Path) -> Result<()> {
        self.pending_sources.insert((url.to_string(), path.to_path_buf()));
        Ok(())
    }

    fn flush_pending_sources(&mut self) -> Result<()> {
        if self.pending_sources.is_empty() {
            return Ok(());
        }

        let pb = make_progress_bar(self.pending_sources.len() as u64);

        println!("{} Analyzing {} sources",
                 style(">").dim(), style(self.pending_sources.len()).yellow());
        for (url, path) in self.pending_sources.drain() {
            pb.set_message(&url);
            let mut f = fs::File::open(&path)?;
            let mut contents = String::new();
            try!(f.read_to_string(&mut contents));
            let ty = if sourcemap::is_sourcemap_slice(contents.as_bytes()) {
                SourceType::SourceMap
            } else if path.file_name()
                .and_then(|x| x.to_str())
                .map(|x| x.contains(".min."))
                .unwrap_or(false) ||
                might_be_minified::analyze_str(&contents).is_likely_minified()
            {
                SourceType::MinifiedScript
            } else {
                SourceType::Script
            };

            self.sources.insert(url.clone(), Source {
                url: url.clone(),
                file_path: path.to_path_buf(),
                contents: contents,
                ty: ty,
                skip_upload: false,
                headers: vec![],
                messages: RefCell::new(vec![]),
            });
            pb.inc(1);
        }

        pb.finish_and_clear();

        Ok(())
    }

    fn validate_script(&self, source: &Source) -> Result<()> {
        let reference = sourcemap::locate_sourcemap_reference_slice(source.contents.as_bytes())?;
        if let sourcemap::SourceMapRef::LegacyRef(_) = reference {
            source.warn("encountered a legacy reference".into());
        }
        if let Some(url) = reference.get_url() {
            let full_url = join_url(&source.url, url)?;
            debug!("found sourcemap for {} at {}", &source.url, full_url);
        } else if source.ty == SourceType::MinifiedScript {
            source.error("missing sourcemap!".into());
        } else {
            source.warn("no source map reference".into());
        }
        Ok(())
    }

    fn validate_sourcemap(&self, source: &Source) -> Result<()> {
        match sourcemap::decode_slice(source.contents.as_bytes())? {
            sourcemap::DecodedMap::Regular(sm) => {
                for idx in 0..sm.get_source_count() {
                    let source_url = sm.get_source(idx).unwrap_or("??");
                    if sm.get_source_contents(idx).is_some() ||
                       self.sources.get(source_url).is_some() {
                        debug!("validator found source ({})", source_url);
                    } else {
                        source.warn(format!("missing sourcecode ({})", source_url));
                    }
                }
            }
            sourcemap::DecodedMap::Index(_) => {
                source.warn("encountered indexed sourcemap. We cannot validate those.".into());
            }
        }
        Ok(())
    }

    pub fn dump_log(&self, title: &str) {
        let mut sources: Vec<_> = self.sources.values().collect();
        sources.sort_by_key(|&source| {
            (source.ty, source.url.clone())
        });

        println!("");
        println!("{}", style(title).dim().bold());
        let mut sect = None;

        for source in sources {
            if Some(source.ty) != sect {
                println!("  {}", style(match source.ty {
                    SourceType::Script => "Scripts",
                    SourceType::MinifiedScript => "Minified Scripts",
                    SourceType::SourceMap => "Source Maps",
                }).yellow().bold());
                sect = Some(source.ty);
            }

            if source.skip_upload {
                println!("    {} [skipped separate upload]", &source.url);
            } else {
                if_chain! {
                    if source.ty == SourceType::MinifiedScript;
                    if let Some(sm_ref) = get_sourcemap_reference_from_headers(
                        source.headers
                        .iter()
                        .map(|&(ref k, ref v)| (k, v)));
                    then {
                        println!("    {} (sourcemap at {})",
                                 &source.url, style(sm_ref).cyan());
                    } else {
                        println!("    {}", &source.url);
                    }
                }
            }

            if !source.messages.borrow().is_empty() {
                for msg in source.messages.borrow().iter() {
                    println!("      - {}: {}", style(&msg.0).red(), msg.1);
                }
            }
        }
    }

    /// Validates all sources within.
    pub fn validate_all(&mut self) -> Result<()> {
        self.flush_pending_sources()?;
        let mut sources: Vec<_> = self.sources.iter().map(|x| x.1).collect();
        sources.sort_by_key(|x| &x.url);
        let mut failed = false;

        println!("{} Validating sources",
                 style(">").dim());
        let pb = ProgressBar::new(sources.len() as u64);
        for source in sources.iter() {
            pb.tick();
            match source.ty {
                SourceType::Script |
                SourceType::MinifiedScript => {
                    if let Err(err) = self.validate_script(&source) {
                        source.error(format!("failed to process: {}", err));
                        failed = true;
                    }
                }
                SourceType::SourceMap => {
                    if let Err(err) = self.validate_sourcemap(&source) {
                        source.error(format!("failed to process: {}", err));
                        failed = true;
                    }
                }
            }
            pb.inc(1);
        }
        pb.finish_and_clear();

        if !failed {
            return Ok(());
        }

        self.dump_log("Source Map Validation Report");
        fail!("Encountered problems when validating source maps.");
    }

    /// Automatically rewrite all source maps.
    ///
    /// This inlines sources, flattens indexes and skips individual uploads.
    pub fn rewrite(&mut self, prefixes: &[&str]) -> Result<()> {
        self.flush_pending_sources()?;

        println!("{} Rewriting sources",
                 style(">").dim());
        let pb = ProgressBar::new(self.sources.len() as u64);
        for (_, source) in self.sources.iter_mut() {
            if source.ty != SourceType::SourceMap {
                pb.inc(1);
                continue;
            }
            pb.tick();
            let options = sourcemap::RewriteOptions {
                load_local_source_contents: true,
                strip_prefixes: prefixes,<commit_msg>Report messages better<commit_after>//! Provides sourcemap validation functionality.
use std::fs;
use std::fmt;
use std::io::Read;
use std::cell::RefCell;
use std::collections::{HashSet, HashMap};
use std::iter::FromIterator;
use std::path::{Path, PathBuf};

use url::Url;
use api::{Api, FileContents};

use sourcemap;
use might_be_minified;
use indicatif::{ProgressBar, ProgressStyle, ProgressDrawTarget, Term, style};

use prelude::*;

fn make_progress_bar(len: u64) -> ProgressBar {
    let pb = ProgressBar::new(len);
    pb.set_draw_target(ProgressDrawTarget::to_term(Term::stdout(), None));
    pb.set_style(ProgressStyle::default_bar()
        .template(&format!("{} {{msg}}\n{{wide_bar}} {{pos}}/{{len}}",
                           style(">").cyan())));
    pb
}

fn join_url(base_url: &str, url: &str) -> Result<String> {
    if base_url.starts_with("~/") {
        match Url::parse(&format!("http://{}", base_url))?.join(url) {
            Ok(url) => {
                let rv = url.to_string();
                if rv.starts_with("http://~/") {
                    Ok(format!("~/{}", &rv[9..]))
                } else {
                    Ok(rv)
                }
            }
            Err(x) => fail!(x),
        }
    } else {
        Ok(Url::parse(base_url)?.join(url)?.to_string())
    }
}

fn split_url(url: &str) -> (Option<&str>, &str, Option<&str>) {
    let mut part_iter = url.rsplitn(2, '/');
    let (filename, ext) = part_iter.next()
        .map(|x| {
            let mut fn_iter = x.splitn(2, '.');
            (fn_iter.next(), fn_iter.next())
        })
        .unwrap_or((None, None));
    let path = part_iter.next();
    (path, filename.unwrap_or(""), ext)
}

fn unsplit_url(path: Option<&str>, basename: &str, ext: Option<&str>) -> String {
    let mut rv = String::new();
    if let Some(path) = path {
        rv.push_str(path);
        rv.push('/');
    }
    rv.push_str(basename);
    if let Some(ext) = ext {
        rv.push('.');
        rv.push_str(ext);
    }
    rv
}

pub fn get_sourcemap_reference_from_headers<'a, I: Iterator<Item = (&'a String, &'a String)>>
    (headers: I)
     -> Option<&'a str> {
    for (k, v) in headers {
        let ki = &k.to_lowercase();
        if ki == "sourcemap" || ki == "x-sourcemap" {
            return Some(v.as_str());
        }
    }
    None
}


fn find_sourcemap_reference(sourcemaps: &HashSet<String>, min_url: &str) -> Result<String> {
    // if there is only one sourcemap in total we just assume that's the one.
    // We just need to make sure that we fix up the reference if we need to
    // (eg: ~/ -> /).
    if sourcemaps.len() == 1 {
        return Ok(sourcemap::make_relative_path(
            min_url, sourcemaps.iter().next().unwrap()));
    }

    let map_ext = "map";
    let (path, basename, ext) = split_url(min_url);

    // foo.min.js -> foo.map
    if sourcemaps.contains(&unsplit_url(path, basename, Some("map"))) {
        return Ok(unsplit_url(None, basename, Some("map")));
    }

    if let Some(ext) = ext.as_ref() {
        // foo.min.js -> foo.min.js.map
        let new_ext = format!("{}.{}", ext, map_ext);
        if sourcemaps.contains(&unsplit_url(path, basename, Some(&new_ext))) {
            return Ok(unsplit_url(None, basename, Some(&new_ext)));
        }

        // foo.min.js -> foo.js.map
        if ext.starts_with("min.") {
            let new_ext = format!("{}.{}", &ext[4..], map_ext);
            if sourcemaps.contains(&unsplit_url(path, basename, Some(&new_ext))) {
                return Ok(unsplit_url(None, basename, Some(&new_ext)));
            }
        }

        // foo.min.js -> foo.min.map
        let mut parts: Vec<_> = ext.split('.').collect();
        if parts.len() > 1 {
            let parts_len = parts.len();
            parts[parts_len - 1] = &map_ext;
            let new_ext = parts.join(".");
            if sourcemaps.contains(&unsplit_url(path, basename, Some(&new_ext))) {
                return Ok(unsplit_url(None, basename, Some(&new_ext)));
            }
        }
    }

    fail!("Could not auto-detect referenced sourcemap for {}.",
          min_url);
}


#[derive(PartialEq, Eq, PartialOrd, Ord, Copy, Clone)]
enum SourceType {
    Script,
    MinifiedScript,
    SourceMap,
}


#[derive(PartialEq, Debug)]
enum LogLevel {
    Warning,
    Error,
}

impl fmt::Display for LogLevel {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            LogLevel::Warning => write!(f, "warning"),
            LogLevel::Error => write!(f, "error"),
        }
    }
}

struct Source {
    url: String,
    #[allow(unused)]
    file_path: PathBuf,
    contents: String,
    ty: SourceType,
    skip_upload: bool,
    headers: Vec<(String, String)>,
    messages: RefCell<Vec<(LogLevel, String)>>,
}

pub struct SourceMapProcessor {
    pending_sources: HashSet<(String, PathBuf)>,
    sources: HashMap<String, Source>,
}

impl Source {

    fn log(&self, level: LogLevel, msg: String) {
        self.messages.borrow_mut().push((level, msg));
    }

    fn warn(&self, msg: String) {
        self.log(LogLevel::Warning, msg);
    }

    fn error(&self, msg: String) {
        self.log(LogLevel::Error, msg);
    }
}

impl SourceMapProcessor {
    /// Creates a new sourcemap validator.
    pub fn new() -> SourceMapProcessor {
        SourceMapProcessor {
            pending_sources: HashSet::new(),
            sources: HashMap::new(),
        }
    }

    /// Adds a new file for processing.
    pub fn add(&mut self, url: &str, path: &Path) -> Result<()> {
        self.pending_sources.insert((url.to_string(), path.to_path_buf()));
        Ok(())
    }

    fn flush_pending_sources(&mut self) -> Result<()> {
        if self.pending_sources.is_empty() {
            return Ok(());
        }

        let pb = make_progress_bar(self.pending_sources.len() as u64);

        println!("{} Analyzing {} sources",
                 style(">").dim(), style(self.pending_sources.len()).yellow());
        for (url, path) in self.pending_sources.drain() {
            pb.set_message(&url);
            let mut f = fs::File::open(&path)?;
            let mut contents = String::new();
            try!(f.read_to_string(&mut contents));
            let ty = if sourcemap::is_sourcemap_slice(contents.as_bytes()) {
                SourceType::SourceMap
            } else if path.file_name()
                .and_then(|x| x.to_str())
                .map(|x| x.contains(".min."))
                .unwrap_or(false) ||
                might_be_minified::analyze_str(&contents).is_likely_minified()
            {
                SourceType::MinifiedScript
            } else {
                SourceType::Script
            };

            self.sources.insert(url.clone(), Source {
                url: url.clone(),
                file_path: path.to_path_buf(),
                contents: contents,
                ty: ty,
                skip_upload: false,
                headers: vec![],
                messages: RefCell::new(vec![]),
            });
            pb.inc(1);
        }

        pb.finish_and_clear();

        Ok(())
    }

    fn validate_script(&self, source: &Source) -> Result<()> {
        let reference = sourcemap::locate_sourcemap_reference_slice(source.contents.as_bytes())?;
        if let sourcemap::SourceMapRef::LegacyRef(_) = reference {
            source.warn("encountered a legacy reference".into());
        }
        if let Some(url) = reference.get_url() {
            let full_url = join_url(&source.url, url)?;
            debug!("found sourcemap for {} at {}", &source.url, full_url);
        } else if source.ty == SourceType::MinifiedScript {
            source.error("missing sourcemap!".into());
        } else {
            source.warn("no source map reference".into());
        }
        Ok(())
    }

    fn validate_sourcemap(&self, source: &Source) -> Result<()> {
        match sourcemap::decode_slice(source.contents.as_bytes())? {
            sourcemap::DecodedMap::Regular(sm) => {
                for idx in 0..sm.get_source_count() {
                    let source_url = sm.get_source(idx).unwrap_or("??");
                    if sm.get_source_contents(idx).is_some() ||
                       self.sources.get(source_url).is_some() {
                        debug!("validator found source ({})", source_url);
                    } else {
                        source.warn(format!("missing sourcecode ({})", source_url));
                    }
                }
            }
            sourcemap::DecodedMap::Index(_) => {
                source.warn("encountered indexed sourcemap. We cannot validate those.".into());
            }
        }
        Ok(())
    }

    pub fn dump_log(&self, title: &str) {
        let mut sources: Vec<_> = self.sources.values().collect();
        sources.sort_by_key(|&source| {
            (source.ty, source.url.clone())
        });

        println!("");
        println!("{}", style(title).dim().bold());
        let mut sect = None;

        for source in sources {
            if Some(source.ty) != sect {
                println!("  {}", style(match source.ty {
                    SourceType::Script => "Scripts",
                    SourceType::MinifiedScript => "Minified Scripts",
                    SourceType::SourceMap => "Source Maps",
                }).yellow().bold());
                sect = Some(source.ty);
            }

            if source.skip_upload {
                println!("    {} [skipped separate upload]", &source.url);
            } else {
                if_chain! {
                    if source.ty == SourceType::MinifiedScript;
                    if let Some(sm_ref) = get_sourcemap_reference_from_headers(
                        source.headers
                        .iter()
                        .map(|&(ref k, ref v)| (k, v)));
                    then {
                        println!("    {} (sourcemap at {})",
                                 &source.url, style(sm_ref).cyan());
                    } else {
                        println!("    {}", &source.url);
                    }
                }
            }

            if !source.messages.borrow().is_empty() {
                for msg in source.messages.borrow().iter() {
                    println!("      - {}: {}", style(&msg.0).red(), msg.1);
                }
            }
        }
    }

    /// Validates all sources within.
    pub fn validate_all(&mut self) -> Result<()> {
        self.flush_pending_sources()?;
        let mut sources: Vec<_> = self.sources.iter().map(|x| x.1).collect();
        sources.sort_by_key(|x| &x.url);
        let mut failed = false;

        println!("{} Validating sources",
                 style(">").dim());
        let pb = make_progress_bar(sources.len() as u64);
        for source in sources.iter() {
            pb.set_message(&source.url);
            match source.ty {
                SourceType::Script |
                SourceType::MinifiedScript => {
                    if let Err(err) = self.validate_script(&source) {
                        source.error(format!("failed to process: {}", err));
                        failed = true;
                    }
                }
                SourceType::SourceMap => {
                    if let Err(err) = self.validate_sourcemap(&source) {
                        source.error(format!("failed to process: {}", err));
                        failed = true;
                    }
                }
            }
            pb.inc(1);
        }
        pb.finish_and_clear();

        if !failed {
            return Ok(());
        }

        self.dump_log("Source Map Validation Report");
        fail!("Encountered problems when validating source maps.");
    }

    /// Automatically rewrite all source maps.
    ///
    /// This inlines sources, flattens indexes and skips individual uploads.
    pub fn rewrite(&mut self, prefixes: &[&str]) -> Result<()> {
        self.flush_pending_sources()?;

        println!("{} Rewriting sources",
                 style(">").dim());
        let pb = make_progress_bar(self.sources.len() as u64);
        for (_, source) in self.sources.iter_mut() {
            pb.set_message(&source.url);
            if source.ty != SourceType::SourceMap {
                pb.inc(1);
                continue;
            }
            let options = sourcemap::RewriteOptions {
                load_local_source_contents: true,
                strip_prefixes: prefixes,
{'fraction_non_alphanumeric': 0.1537229142468058, 'fraction_numerical': 0.001762326270677294, 'mean_word_length': 1.7831902797904358, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 121, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3704117', 'n_tokens_mistral': 7503, 'n_tokens_neox': 7324, 'n_words': 1539}