<commit_before>use crate::syntax::discriminant::DiscriminantSet;
use crate::syntax::report::Errors;
use crate::syntax::Atom::*;
use crate::syntax::{
    attrs, error, Api, Doc, Enum, ExternFn, ExternType, Lang, Receiver, Ref, Signature, Slice,
    Struct, Ty1, Type, TypeAlias, Var, Variant,
};
use proc_macro2::TokenStream;
use quote::{format_ident, quote, quote_spanned};
use syn::parse::{ParseStream, Parser};
use syn::punctuated::Punctuated;
use syn::{
    Abi, Attribute, Error, Fields, FnArg, ForeignItem, ForeignItemFn, ForeignItemType,
    GenericArgument, Ident, Item, ItemEnum, ItemForeignMod, ItemStruct, Pat, PathArguments, Result,
    ReturnType, Token, Type as RustType, TypeBareFn, TypePath, TypeReference, TypeSlice,
};

pub mod kw {
    syn::custom_keyword!(Result);
}

pub fn parse_items(cx: &mut Errors, items: Vec<Item>) -> Vec<Api> {
    let mut apis = Vec::new();
    for item in items {
        match item {
            Item::Struct(item) => match parse_struct(cx, item) {
                Ok(strct) => apis.push(strct),
                Err(err) => cx.push(err),
            },
            Item::Enum(item) => match parse_enum(cx, item) {
                Ok(enm) => apis.push(enm),
                Err(err) => cx.push(err),
            },
            Item::ForeignMod(foreign_mod) => parse_foreign_mod(cx, foreign_mod, &mut apis),
            Item::Use(item) => cx.error(item, error::USE_NOT_ALLOWED),
            _ => cx.error(item, "unsupported item"),
        }
    }
    apis
}

fn parse_struct(cx: &mut Errors, item: ItemStruct) -> Result<Api> {
    let generics = &item.generics;
    if !generics.params.is_empty() || generics.where_clause.is_some() {
        let struct_token = item.struct_token;
        let ident = &item.ident;
        let where_clause = &generics.where_clause;
        let span = quote!(#struct_token #ident #generics #where_clause);
        return Err(Error::new_spanned(
            span,
            "struct with generic parameters is not supported yet",
        ));
    }

    let mut doc = Doc::new();
    let mut derives = Vec::new();
    attrs::parse(
        cx,
        &item.attrs,
        attrs::Parser {
            doc: Some(&mut doc),
            derives: Some(&mut derives),
        },
    );

    let fields = match item.fields {
        Fields::Named(fields) => fields,
        Fields::Unit => return Err(Error::new_spanned(item, "unit structs are not supported")),
        Fields::Unnamed(_) => {
            return Err(Error::new_spanned(item, "tuple structs are not supported"))
        }
    };

    Ok(Api::Struct(Struct {
        doc,
        derives,
        struct_token: item.struct_token,
        ident: item.ident,
        brace_token: fields.brace_token,
        fields: fields
            .named
            .into_iter()
            .map(|field| {
                Ok(Var {
                    ident: field.ident.unwrap(),
                    ty: parse_type(&field.ty)?,
                })
            })
            .collect::<Result<_>>()?,
    }))
}

fn parse_enum(cx: &mut Errors, item: ItemEnum) -> Result<Api> {
    let generics = &item.generics;
    if !generics.params.is_empty() || generics.where_clause.is_some() {
        let enum_token = item.enum_token;
        let ident = &item.ident;
        let where_clause = &generics.where_clause;
        let span = quote!(#enum_token #ident #generics #where_clause);
        return Err(Error::new_spanned(
            span,
            "enums with generic parameters are not allowed",
        ));
    }

    let doc = attrs::parse_doc(cx, &item.attrs);

    let mut variants = Vec::new();
    let mut discriminants = DiscriminantSet::new();
    for variant in item.variants {
        match variant.fields {
            Fields::Unit => {}
            _ => {
                return Err(Error::new_spanned(
                    variant,
                    "enums with data are not supported yet",
                ));
            }
        }
        let expr = variant.discriminant.as_ref().map(|(_, expr)| expr);
        let try_discriminant = match &expr {
            Some(lit) => discriminants.insert(lit),
            None => discriminants.insert_next(),
        };
        let discriminant = match try_discriminant {
            Ok(discriminant) => discriminant,
            Err(err) => return Err(Error::new_spanned(variant, err)),
        };
        let expr = variant.discriminant.map(|(_, expr)| expr);
        variants.push(Variant {
            ident: variant.ident,
            discriminant,
            expr,
        });
    }

    let enum_token = item.enum_token;
    let brace_token = item.brace_token;

    let repr = match discriminants.inferred_repr() {
        Ok(repr) => repr,
        Err(err) => {
            let span = quote_spanned!(brace_token.span=> #enum_token {});
            return Err(Error::new_spanned(span, err));
        }
    };

    Ok(Api::Enum(Enum {
        doc,
        enum_token,
        ident: item.ident,
        brace_token,
        variants,
        repr,
    }))
}

fn parse_foreign_mod(cx: &mut Errors, foreign_mod: ItemForeignMod, out: &mut Vec<Api>) {
    let lang = match parse_lang(foreign_mod.abi) {
        Ok(lang) => lang,
        Err(err) => return cx.push(err),
    };

    let mut items = Vec::new();
    for foreign in &foreign_mod.items {
        match foreign {
            ForeignItem::Type(foreign) => match parse_extern_type(cx, foreign, lang) {
                Ok(ety) => items.push(ety),
                Err(err) => cx.push(err),
            },
            ForeignItem::Fn(foreign) => match parse_extern_fn(cx, foreign, lang) {
                Ok(efn) => items.push(efn),
                Err(err) => cx.push(err),<commit_msg>Recover from some enum parsing errors<commit_after>use crate::syntax::discriminant::DiscriminantSet;
use crate::syntax::report::Errors;
use crate::syntax::Atom::*;
use crate::syntax::{
    attrs, error, Api, Doc, Enum, ExternFn, ExternType, Lang, Receiver, Ref, Signature, Slice,
    Struct, Ty1, Type, TypeAlias, Var, Variant,
};
use proc_macro2::TokenStream;
use quote::{format_ident, quote, quote_spanned};
use syn::parse::{ParseStream, Parser};
use syn::punctuated::Punctuated;
use syn::{
    Abi, Attribute, Error, Fields, FnArg, ForeignItem, ForeignItemFn, ForeignItemType,
    GenericArgument, Ident, Item, ItemEnum, ItemForeignMod, ItemStruct, Pat, PathArguments, Result,
    ReturnType, Token, Type as RustType, TypeBareFn, TypePath, TypeReference, TypeSlice,
};

pub mod kw {
    syn::custom_keyword!(Result);
}

pub fn parse_items(cx: &mut Errors, items: Vec<Item>) -> Vec<Api> {
    let mut apis = Vec::new();
    for item in items {
        match item {
            Item::Struct(item) => match parse_struct(cx, item) {
                Ok(strct) => apis.push(strct),
                Err(err) => cx.push(err),
            },
            Item::Enum(item) => match parse_enum(cx, item) {
                Ok(enm) => apis.push(enm),
                Err(err) => cx.push(err),
            },
            Item::ForeignMod(foreign_mod) => parse_foreign_mod(cx, foreign_mod, &mut apis),
            Item::Use(item) => cx.error(item, error::USE_NOT_ALLOWED),
            _ => cx.error(item, "unsupported item"),
        }
    }
    apis
}

fn parse_struct(cx: &mut Errors, item: ItemStruct) -> Result<Api> {
    let generics = &item.generics;
    if !generics.params.is_empty() || generics.where_clause.is_some() {
        let struct_token = item.struct_token;
        let ident = &item.ident;
        let where_clause = &generics.where_clause;
        let span = quote!(#struct_token #ident #generics #where_clause);
        return Err(Error::new_spanned(
            span,
            "struct with generic parameters is not supported yet",
        ));
    }

    let mut doc = Doc::new();
    let mut derives = Vec::new();
    attrs::parse(
        cx,
        &item.attrs,
        attrs::Parser {
            doc: Some(&mut doc),
            derives: Some(&mut derives),
        },
    );

    let fields = match item.fields {
        Fields::Named(fields) => fields,
        Fields::Unit => return Err(Error::new_spanned(item, "unit structs are not supported")),
        Fields::Unnamed(_) => {
            return Err(Error::new_spanned(item, "tuple structs are not supported"))
        }
    };

    Ok(Api::Struct(Struct {
        doc,
        derives,
        struct_token: item.struct_token,
        ident: item.ident,
        brace_token: fields.brace_token,
        fields: fields
            .named
            .into_iter()
            .map(|field| {
                Ok(Var {
                    ident: field.ident.unwrap(),
                    ty: parse_type(&field.ty)?,
                })
            })
            .collect::<Result<_>>()?,
    }))
}

fn parse_enum(cx: &mut Errors, item: ItemEnum) -> Result<Api> {
    let generics = &item.generics;
    if !generics.params.is_empty() || generics.where_clause.is_some() {
        let enum_token = item.enum_token;
        let ident = &item.ident;
        let where_clause = &generics.where_clause;
        let span = quote!(#enum_token #ident #generics #where_clause);
        return Err(Error::new_spanned(
            span,
            "enums with generic parameters are not allowed",
        ));
    }

    let doc = attrs::parse_doc(cx, &item.attrs);

    let mut variants = Vec::new();
    let mut discriminants = DiscriminantSet::new();
    for variant in item.variants {
        match variant.fields {
            Fields::Unit => {}
            _ => {
                cx.error(variant, "enums with data are not supported yet");
                break;
            }
        }
        let expr = variant.discriminant.as_ref().map(|(_, expr)| expr);
        let try_discriminant = match &expr {
            Some(lit) => discriminants.insert(lit),
            None => discriminants.insert_next(),
        };
        let discriminant = match try_discriminant {
            Ok(discriminant) => discriminant,
            Err(err) => {
                cx.error(variant, err);
                break;
            }
        };
        let expr = variant.discriminant.map(|(_, expr)| expr);
        variants.push(Variant {
            ident: variant.ident,
            discriminant,
            expr,
        });
    }

    let enum_token = item.enum_token;
    let brace_token = item.brace_token;

    let mut repr = U8;
    match discriminants.inferred_repr() {
        Ok(inferred) => repr = inferred,
        Err(err) => {
            let span = quote_spanned!(brace_token.span=> #enum_token {});
            cx.error(span, err);
            variants.clear();
        }
    }

    Ok(Api::Enum(Enum {
        doc,
        enum_token,
        ident: item.ident,
        brace_token,
        variants,
        repr,
    }))
}

fn parse_foreign_mod(cx: &mut Errors, foreign_mod: ItemForeignMod, out: &mut Vec<Api>) {
    let lang = match parse_lang(foreign_mod.abi) {
        Ok(lang) => lang,
        Err(err) => return cx.push(err),
    };

    let mut items = Vec::new();
    for foreign in &foreign_mod.items {
        match foreign {
            ForeignItem::Type(foreign) => match parse_extern_type(cx, foreign, lang) {
                Ok(ety) => items.push(ety),
                Err(err) => cx.push(err),
            },
            ForeignItem::Fn(foreign) => match parse_extern_fn(cx, foreign, lang) {
                Ok(efn) => items.push(efn),
                Err(err) => cx.push(err),
{'fraction_non_alphanumeric': 0.13280501611077244, 'fraction_numerical': 0.00043542628233040145, 'mean_word_length': 2.115572436245252, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '756260', 'n_tokens_mistral': 3580, 'n_tokens_neox': 3456, 'n_words': 782}