<commit_before>use std::cell::RefCell;
use std::default::Default;
use std::fmt;
use std::hash::{Hash, Hasher};
use std::iter::FromIterator;
use std::lazy::SyncOnceCell as OnceCell;
use std::rc::Rc;
use std::sync::Arc;
use std::{slice, vec};

use arrayvec::ArrayVec;
use rustc_ast::attr;
use rustc_ast::util::comments::beautify_doc_string;
use rustc_ast::{self as ast, AttrStyle};
use rustc_attr::{ConstStability, Deprecation, Stability, StabilityLevel};
use rustc_data_structures::fx::{FxHashMap, FxHashSet};
use rustc_feature::UnstableFeatures;
use rustc_hir as hir;
use rustc_hir::def::{CtorKind, Res};
use rustc_hir::def_id::{CrateNum, DefId, DefIndex};
use rustc_hir::lang_items::LangItem;
use rustc_hir::{BodyId, Mutability};
use rustc_index::vec::IndexVec;
use rustc_middle::ty::{self, TyCtxt};
use rustc_session::Session;
use rustc_span::hygiene::MacroKind;
use rustc_span::source_map::DUMMY_SP;
use rustc_span::symbol::{kw, sym, Ident, Symbol, SymbolStr};
use rustc_span::{self, FileName, Loc};
use rustc_target::abi::VariantIdx;
use rustc_target::spec::abi::Abi;

use crate::clean::cfg::Cfg;
use crate::clean::external_path;
use crate::clean::inline;
use crate::clean::types::Type::{QPath, ResolvedPath};
use crate::clean::Clean;
use crate::core::DocContext;
use crate::formats::cache::Cache;
use crate::formats::item_type::ItemType;
use crate::html::render::cache::ExternalLocation;

use self::FnRetTy::*;
use self::ItemKind::*;
use self::SelfTy::*;
use self::Type::*;

thread_local!(crate static MAX_DEF_IDX: RefCell<FxHashMap<CrateNum, DefIndex>> = Default::default());

#[derive(Clone, Debug)]
crate struct Crate {
    crate name: Symbol,
    crate version: Option<String>,
    crate src: FileName,
    crate module: Option<Item>,
    crate externs: Vec<(CrateNum, ExternalCrate)>,
    crate primitives: Vec<(DefId, PrimitiveType)>,
    // These are later on moved into `CACHEKEY`, leaving the map empty.
    // Only here so that they can be filtered through the rustdoc passes.
    crate external_traits: Rc<RefCell<FxHashMap<DefId, Trait>>>,
    crate masked_crates: FxHashSet<CrateNum>,
    crate collapsed: bool,
}

#[derive(Clone, Debug)]
crate struct ExternalCrate {
    crate name: Symbol,
    crate src: FileName,
    crate attrs: Attributes,
    crate primitives: Vec<(DefId, PrimitiveType)>,
    crate keywords: Vec<(DefId, Symbol)>,
}

/// Anything with a source location and set of attributes and, optionally, a
/// name. That is, anything that can be documented. This doesn't correspond
/// directly to the AST's concept of an item; it's a strict superset.
#[derive(Clone)]
crate struct Item {
    /// Stringified span
    crate source: Span,
    /// Not everything has a name. E.g., impls
    crate name: Option<Symbol>,
    crate attrs: Box<Attributes>,
    crate visibility: Visibility,
    crate kind: Box<ItemKind>,
    crate def_id: DefId,
}

// `Item` is used a lot. Make sure it doesn't unintentionally get bigger.
#[cfg(target_arch = "x86_64")]
rustc_data_structures::static_assert_size!(Item, 48);

impl fmt::Debug for Item {
    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
        let def_id: &dyn fmt::Debug = if self.is_fake() { &"**FAKE**" } else { &self.def_id };

        fmt.debug_struct("Item")
            .field("source", &self.source)
            .field("name", &self.name)
            .field("attrs", &self.attrs)
            .field("kind", &self.kind)
            .field("visibility", &self.visibility)
            .field("def_id", def_id)
            .finish()
    }
}

impl Item {
    crate fn stability<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Option<&'tcx Stability> {
        if self.is_fake() { None } else { tcx.lookup_stability(self.def_id) }
    }

    crate fn const_stability<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Option<&'tcx ConstStability> {
        if self.is_fake() { None } else { tcx.lookup_const_stability(self.def_id) }
    }

    crate fn deprecation(&self, tcx: TyCtxt<'_>) -> Option<Deprecation> {
        if self.is_fake() { None } else { tcx.lookup_deprecation(self.def_id) }
    }

    /// Finds the `doc` attribute as a NameValue and returns the corresponding
    /// value found.
    crate fn doc_value(&self) -> Option<String> {
        self.attrs.doc_value()
    }

    /// Convenience wrapper around [`Self::from_def_id_and_parts`] which converts
    /// `hir_id` to a [`DefId`]
    pub fn from_hir_id_and_parts(
        hir_id: hir::HirId,
        name: Option<Symbol>,
        kind: ItemKind,
        cx: &mut DocContext<'_>,
    ) -> Item {
        Item::from_def_id_and_parts(cx.tcx.hir().local_def_id(hir_id).to_def_id(), name, kind, cx)
    }

    pub fn from_def_id_and_parts(
        def_id: DefId,
        name: Option<Symbol>,
        kind: ItemKind,
        cx: &mut DocContext<'_>,
    ) -> Item {
        Self::from_def_id_and_attrs_and_parts(
            def_id,
            name,
            kind,
            box cx.tcx.get_attrs(def_id).clean(cx),
            cx,
        )
    }

    pub fn from_def_id_and_attrs_and_parts(
        def_id: DefId,
        name: Option<Symbol>,
        kind: ItemKind,
        attrs: Box<Attributes>,
        cx: &DocContext<'_>,
    ) -> Item {
        debug!("name={:?}, def_id={:?}", name, def_id);

        // `span_if_local()` lies about functions and only gives the span of the function signature
        let source = def_id.as_local().map_or_else(
            || cx.tcx.def_span(def_id),
            |local| {
                let hir = cx.tcx.hir();
                hir.span_with_body(hir.local_def_id_to_hir_id(local))
            },
        );

        Item {
            def_id,
            kind: box kind,
            name,
            source: source.clean(cx),
            attrs,
            visibility: cx.tcx.visibility(def_id).clean(cx),<commit_msg>Make DocContext &mut for clean<commit_after>use std::cell::RefCell;
use std::default::Default;
use std::fmt;
use std::hash::{Hash, Hasher};
use std::iter::FromIterator;
use std::lazy::SyncOnceCell as OnceCell;
use std::rc::Rc;
use std::sync::Arc;
use std::{slice, vec};

use arrayvec::ArrayVec;
use rustc_ast::attr;
use rustc_ast::util::comments::beautify_doc_string;
use rustc_ast::{self as ast, AttrStyle};
use rustc_attr::{ConstStability, Deprecation, Stability, StabilityLevel};
use rustc_data_structures::fx::{FxHashMap, FxHashSet};
use rustc_feature::UnstableFeatures;
use rustc_hir as hir;
use rustc_hir::def::{CtorKind, Res};
use rustc_hir::def_id::{CrateNum, DefId, DefIndex};
use rustc_hir::lang_items::LangItem;
use rustc_hir::{BodyId, Mutability};
use rustc_index::vec::IndexVec;
use rustc_middle::ty::{self, TyCtxt};
use rustc_session::Session;
use rustc_span::hygiene::MacroKind;
use rustc_span::source_map::DUMMY_SP;
use rustc_span::symbol::{kw, sym, Ident, Symbol, SymbolStr};
use rustc_span::{self, FileName, Loc};
use rustc_target::abi::VariantIdx;
use rustc_target::spec::abi::Abi;

use crate::clean::cfg::Cfg;
use crate::clean::external_path;
use crate::clean::inline;
use crate::clean::types::Type::{QPath, ResolvedPath};
use crate::clean::Clean;
use crate::core::DocContext;
use crate::formats::cache::Cache;
use crate::formats::item_type::ItemType;
use crate::html::render::cache::ExternalLocation;

use self::FnRetTy::*;
use self::ItemKind::*;
use self::SelfTy::*;
use self::Type::*;

thread_local!(crate static MAX_DEF_IDX: RefCell<FxHashMap<CrateNum, DefIndex>> = Default::default());

#[derive(Clone, Debug)]
crate struct Crate {
    crate name: Symbol,
    crate version: Option<String>,
    crate src: FileName,
    crate module: Option<Item>,
    crate externs: Vec<(CrateNum, ExternalCrate)>,
    crate primitives: Vec<(DefId, PrimitiveType)>,
    // These are later on moved into `CACHEKEY`, leaving the map empty.
    // Only here so that they can be filtered through the rustdoc passes.
    crate external_traits: Rc<RefCell<FxHashMap<DefId, Trait>>>,
    crate masked_crates: FxHashSet<CrateNum>,
    crate collapsed: bool,
}

#[derive(Clone, Debug)]
crate struct ExternalCrate {
    crate name: Symbol,
    crate src: FileName,
    crate attrs: Attributes,
    crate primitives: Vec<(DefId, PrimitiveType)>,
    crate keywords: Vec<(DefId, Symbol)>,
}

/// Anything with a source location and set of attributes and, optionally, a
/// name. That is, anything that can be documented. This doesn't correspond
/// directly to the AST's concept of an item; it's a strict superset.
#[derive(Clone)]
crate struct Item {
    /// Stringified span
    crate source: Span,
    /// Not everything has a name. E.g., impls
    crate name: Option<Symbol>,
    crate attrs: Box<Attributes>,
    crate visibility: Visibility,
    crate kind: Box<ItemKind>,
    crate def_id: DefId,
}

// `Item` is used a lot. Make sure it doesn't unintentionally get bigger.
#[cfg(target_arch = "x86_64")]
rustc_data_structures::static_assert_size!(Item, 48);

impl fmt::Debug for Item {
    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
        let def_id: &dyn fmt::Debug = if self.is_fake() { &"**FAKE**" } else { &self.def_id };

        fmt.debug_struct("Item")
            .field("source", &self.source)
            .field("name", &self.name)
            .field("attrs", &self.attrs)
            .field("kind", &self.kind)
            .field("visibility", &self.visibility)
            .field("def_id", def_id)
            .finish()
    }
}

impl Item {
    crate fn stability<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Option<&'tcx Stability> {
        if self.is_fake() { None } else { tcx.lookup_stability(self.def_id) }
    }

    crate fn const_stability<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Option<&'tcx ConstStability> {
        if self.is_fake() { None } else { tcx.lookup_const_stability(self.def_id) }
    }

    crate fn deprecation(&self, tcx: TyCtxt<'_>) -> Option<Deprecation> {
        if self.is_fake() { None } else { tcx.lookup_deprecation(self.def_id) }
    }

    /// Finds the `doc` attribute as a NameValue and returns the corresponding
    /// value found.
    crate fn doc_value(&self) -> Option<String> {
        self.attrs.doc_value()
    }

    /// Convenience wrapper around [`Self::from_def_id_and_parts`] which converts
    /// `hir_id` to a [`DefId`]
    pub fn from_hir_id_and_parts(
        hir_id: hir::HirId,
        name: Option<Symbol>,
        kind: ItemKind,
        cx: &mut DocContext<'_>,
    ) -> Item {
        Item::from_def_id_and_parts(cx.tcx.hir().local_def_id(hir_id).to_def_id(), name, kind, cx)
    }

    pub fn from_def_id_and_parts(
        def_id: DefId,
        name: Option<Symbol>,
        kind: ItemKind,
        cx: &mut DocContext<'_>,
    ) -> Item {
        Self::from_def_id_and_attrs_and_parts(
            def_id,
            name,
            kind,
            box cx.tcx.get_attrs(def_id).clean(cx),
            cx,
        )
    }

    pub fn from_def_id_and_attrs_and_parts(
        def_id: DefId,
        name: Option<Symbol>,
        kind: ItemKind,
        attrs: Box<Attributes>,
        cx: &mut DocContext<'_>,
    ) -> Item {
        debug!("name={:?}, def_id={:?}", name, def_id);

        // `span_if_local()` lies about functions and only gives the span of the function signature
        let source = def_id.as_local().map_or_else(
            || cx.tcx.def_span(def_id),
            |local| {
                let hir = cx.tcx.hir();
                hir.span_with_body(hir.local_def_id_to_hir_id(local))
            },
        );

        Item {
            def_id,
            kind: box kind,
            name,
            source: source.clean(cx),
            attrs,
            visibility: cx.tcx.visibility(def_id).clean(cx),
{'fraction_non_alphanumeric': 0.14356989247311827, 'fraction_numerical': 0.001032258064516129, 'mean_word_length': 3.3251488095238093, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 83, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2987782', 'n_tokens_mistral': 3908, 'n_tokens_neox': 3983, 'n_words': 934}