<commit_before>use super::*;

// test expr_literals
// fn foo() {
//     let _ = true;
//     let _ = false;
//     let _ = 1;
//     let _ = 2.0;
//     let _ = b'a';
//     let _ = 'b';
//     let _ = "c";
//     let _ = r"d";
//     let _ = b"e";
//     let _ = br"f";
// }
pub(crate) const LITERAL_FIRST: TokenSet = token_set![
    TRUE_KW,
    FALSE_KW,
    INT_NUMBER,
    FLOAT_NUMBER,
    BYTE,
    CHAR,
    STRING,
    RAW_STRING,
    BYTE_STRING,
    RAW_BYTE_STRING
];

pub(crate) fn literal(p: &mut Parser) -> Option<CompletedMarker> {
    if !p.at_ts(LITERAL_FIRST) {
        return None;
    }
    let m = p.start();
    p.bump();
    Some(m.complete(p, LITERAL))
}

// E.g. for after the break in `if break {}`, this should not match
pub(super) const ATOM_EXPR_FIRST: TokenSet = LITERAL_FIRST.union(token_set![
    L_PAREN,
    L_CURLY,
    L_BRACK,
    L_ANGLE, // Universal function call syntax
    PIPE,
    MOVE_KW,
    IF_KW,
    WHILE_KW,
    MATCH_KW,
    UNSAFE_KW,
    RETURN_KW,
    IDENT,
    SELF_KW,
    SUPER_KW,
    CRATE_KW,
    COLONCOLON,
    BREAK_KW,
    CONTINUE_KW,
    LIFETIME,
]);

const EXPR_RECOVERY_SET: TokenSet = token_set![LET_KW];

pub(super) fn atom_expr(p: &mut Parser, r: Restrictions) -> Option<(CompletedMarker, BlockLike)> {
    if let Some(m) = literal(p) {
        return Some((m, BlockLike::NotBlock));
    }
    if paths::is_path_start(p) || p.at(L_ANGLE) {
        return Some(path_expr(p, r));
    }
    let la = p.nth(1);
    let done = match p.current() {
        L_PAREN => tuple_expr(p),
        L_BRACK => array_expr(p),
        PIPE => lambda_expr(p),
        MOVE_KW if la == PIPE => lambda_expr(p),
        IF_KW => if_expr(p),

        LOOP_KW => loop_expr(p, None),
        FOR_KW => for_expr(p, None),
        WHILE_KW => while_expr(p, None),
        LIFETIME if la == COLON => {
            let m = p.start();<commit_msg>Dedupe PATH_FIRST in ATOM_EXPR_FIRST<commit_after>use super::*;
use crate::grammar::paths::PATH_FIRST;

// test expr_literals
// fn foo() {
//     let _ = true;
//     let _ = false;
//     let _ = 1;
//     let _ = 2.0;
//     let _ = b'a';
//     let _ = 'b';
//     let _ = "c";
//     let _ = r"d";
//     let _ = b"e";
//     let _ = br"f";
// }
pub(crate) const LITERAL_FIRST: TokenSet = token_set![
    TRUE_KW,
    FALSE_KW,
    INT_NUMBER,
    FLOAT_NUMBER,
    BYTE,
    CHAR,
    STRING,
    RAW_STRING,
    BYTE_STRING,
    RAW_BYTE_STRING
];

pub(crate) fn literal(p: &mut Parser) -> Option<CompletedMarker> {
    if !p.at_ts(LITERAL_FIRST) {
        return None;
    }
    let m = p.start();
    p.bump();
    Some(m.complete(p, LITERAL))
}

// E.g. for after the break in `if break {}`, this should not match
pub(super) const ATOM_EXPR_FIRST: TokenSet = LITERAL_FIRST.union(PATH_FIRST).union(token_set![
    L_PAREN,
    L_CURLY,
    L_BRACK,
    PIPE,
    MOVE_KW,
    IF_KW,
    WHILE_KW,
    MATCH_KW,
    UNSAFE_KW,
    RETURN_KW,
    BREAK_KW,
    CONTINUE_KW,
    LIFETIME,
]);

const EXPR_RECOVERY_SET: TokenSet = token_set![LET_KW];

pub(super) fn atom_expr(p: &mut Parser, r: Restrictions) -> Option<(CompletedMarker, BlockLike)> {
    if let Some(m) = literal(p) {
        return Some((m, BlockLike::NotBlock));
    }
    if paths::is_path_start(p) || p.at(L_ANGLE) {
        return Some(path_expr(p, r));
    }
    let la = p.nth(1);
    let done = match p.current() {
        L_PAREN => tuple_expr(p),
        L_BRACK => array_expr(p),
        PIPE => lambda_expr(p),
        MOVE_KW if la == PIPE => lambda_expr(p),
        IF_KW => if_expr(p),

        LOOP_KW => loop_expr(p, None),
        FOR_KW => for_expr(p, None),
        WHILE_KW => while_expr(p, None),
        LIFETIME if la == COLON => {
            let m = p.start();
{'fraction_non_alphanumeric': 0.15035990402559318, 'fraction_numerical': 0.0021327645961077045, 'mean_word_length': 2.268292682926829, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3427104', 'n_tokens_mistral': 1566, 'n_tokens_neox': 1519, 'n_words': 312}