<commit_before>use crate::clean;
use crate::docfs::PathError;
use crate::error::Error;
use crate::fold::DocFolder;
use crate::html::format::Buffer;
use crate::html::highlight;
use crate::html::layout;
use crate::html::render::{SharedContext, BASIC_KEYWORDS};
use rustc_hir::def_id::LOCAL_CRATE;
use rustc_session::Session;
use rustc_span::edition::Edition;
use rustc_span::source_map::FileName;
use std::ffi::OsStr;
use std::fs;
use std::path::{Component, Path, PathBuf};

crate fn render(
    dst: &Path,
    scx: &mut SharedContext<'_>,
    krate: clean::Crate,
) -> Result<clean::Crate, Error> {
    info!("emitting source files");
    let dst = dst.join("src").join(&*krate.name.as_str());
    scx.ensure_dir(&dst)?;
    let mut folder = SourceCollector { dst, scx };
    Ok(folder.fold_crate(krate))
}

/// Helper struct to render all source code to HTML pages
struct SourceCollector<'a, 'tcx> {
    scx: &'a mut SharedContext<'tcx>,

    /// Root destination to place all HTML output into
    dst: PathBuf,
}

impl DocFolder for SourceCollector<'_, '_> {
    fn fold_item(&mut self, item: clean::Item) -> Option<clean::Item> {
        // If we're not rendering sources, there's nothing to do.
        // If we're including source files, and we haven't seen this file yet,
        // then we need to render it out to the filesystem.
        if self.scx.include_sources
            // skip all synthetic "files"
            && item.span.filename(self.sess()).is_real()
            // skip non-local files
            && item.span.cnum(self.sess()) == LOCAL_CRATE
        {
            let filename = item.span.filename(self.sess());
            // If it turns out that we couldn't read this file, then we probably
            // can't read any of the files (generating html output from json or
            // something like that), so just don't include sources for the
            // entire crate. The other option is maintaining this mapping on a
            // per-file basis, but that's probably not worth it...
            self.scx.include_sources = match self.emit_source(&filename) {
                Ok(()) => true,
                Err(e) => {
                    println!(
                        "warning: source code was requested to be rendered, \
                         but processing `{}` had an error: {}",
                        filename, e
                    );
                    println!("         skipping rendering of source code");
                    false
                }
            };
        }
        // FIXME: if `include_sources` isn't set and DocFolder didn't require consuming the crate by value,
        // we could return None here without having to walk the rest of the crate.
        Some(self.fold_item_recur(item))<commit_msg>Rollup merge of #82993 - camelid:source-use-diag, r=jyn514<commit_after>use crate::clean;
use crate::docfs::PathError;
use crate::error::Error;
use crate::fold::DocFolder;
use crate::html::format::Buffer;
use crate::html::highlight;
use crate::html::layout;
use crate::html::render::{SharedContext, BASIC_KEYWORDS};
use rustc_hir::def_id::LOCAL_CRATE;
use rustc_session::Session;
use rustc_span::edition::Edition;
use rustc_span::source_map::FileName;
use std::ffi::OsStr;
use std::fs;
use std::path::{Component, Path, PathBuf};

crate fn render(
    dst: &Path,
    scx: &mut SharedContext<'_>,
    krate: clean::Crate,
) -> Result<clean::Crate, Error> {
    info!("emitting source files");
    let dst = dst.join("src").join(&*krate.name.as_str());
    scx.ensure_dir(&dst)?;
    let mut folder = SourceCollector { dst, scx };
    Ok(folder.fold_crate(krate))
}

/// Helper struct to render all source code to HTML pages
struct SourceCollector<'a, 'tcx> {
    scx: &'a mut SharedContext<'tcx>,

    /// Root destination to place all HTML output into
    dst: PathBuf,
}

impl DocFolder for SourceCollector<'_, '_> {
    fn fold_item(&mut self, item: clean::Item) -> Option<clean::Item> {
        // If we're not rendering sources, there's nothing to do.
        // If we're including source files, and we haven't seen this file yet,
        // then we need to render it out to the filesystem.
        if self.scx.include_sources
            // skip all synthetic "files"
            && item.span.filename(self.sess()).is_real()
            // skip non-local files
            && item.span.cnum(self.sess()) == LOCAL_CRATE
        {
            let filename = item.span.filename(self.sess());
            // If it turns out that we couldn't read this file, then we probably
            // can't read any of the files (generating html output from json or
            // something like that), so just don't include sources for the
            // entire crate. The other option is maintaining this mapping on a
            // per-file basis, but that's probably not worth it...
            self.scx.include_sources = match self.emit_source(&filename) {
                Ok(()) => true,
                Err(e) => {
                    self.scx.tcx.sess.span_err(
                        item.span.inner(),
                        &format!("failed to render source code for `{}`: {}", filename, e),
                    );
                    false
                }
            };
        }
        // FIXME: if `include_sources` isn't set and DocFolder didn't require consuming the crate by value,
        // we could return None here without having to walk the rest of the crate.
        Some(self.fold_item_recur(item))
{'fraction_non_alphanumeric': 0.12295081967213115, 'fraction_numerical': 0.0014571948998178506, 'mean_word_length': 2.374923171481254, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2206001', 'n_tokens_mistral': 1593, 'n_tokens_neox': 1562, 'n_words': 500}