<commit_before>use super::cst::*;
use nom::{
    branch::alt,
    bytes::complete::{tag, take_while, take_while_m_n},
    character::complete::{alphanumeric0, anychar, line_ending, not_line_ending, space1},
    combinator::{map, opt, recognize, success, verify},
    multi::{count, many0},
    sequence::{delimited, tuple},
    IResult, Offset, Parser,
};
use nom_supreme::{error::ErrorTree, final_parser::final_parser, ParserExt};
use proptest::prelude::*;

type ParserResult<'a, T> = IResult<&'a str, T, ErrorTree<&'a str>>;

pub trait StringToCst {
    fn parse_cst(&self) -> Vec<Cst>;
}
impl StringToCst for str {
    fn parse_cst(&self) -> Vec<Cst> {
        // TODO: handle trailing whitespace and comments properly
        let source = format!("\n{}", self);
        let parser = |input| expressions0(&source, input, 0);
        let result: Result<_, ErrorTree<&str>> = final_parser(parser)(&source);
        match result {
            Ok(parsed) => fix_offsets_csts(parsed),
            Err(err) => vec![Cst::Error {
                offset: 0,
                unparsable_input: self.to_owned(),
                message: format!("An error occurred while parsing: {:?}", err),
            }],
        }
    }
}

/// Because we don't parse the input directly, but prepend a newline to it, we
/// need to adjust the offsets of the CSTs to account for that.
fn fix_offsets_csts(csts: Vec<Cst>) -> Vec<Cst> {
    csts.into_iter().map(|cst| fix_offsets_cst(cst)).collect()
}
fn fix_offsets_cst(cst: Cst) -> Cst {
    match cst {
        Cst::EqualsSign { offset } => Cst::EqualsSign { offset: offset - 1 },
        Cst::OpeningParenthesis { offset } => Cst::OpeningParenthesis { offset: offset - 1 },<commit_msg>fix(compiler): handle trailing newlines<commit_after>use super::cst::*;
use nom::{
    branch::alt,
    bytes::complete::{tag, take_while, take_while_m_n},
    character::complete::{alphanumeric0, anychar, line_ending, not_line_ending, space1},
    combinator::{map, opt, recognize, success, verify},
    multi::{count, many0},
    sequence::{delimited, tuple},
    IResult, Offset, Parser,
};
use nom_supreme::{error::ErrorTree, final_parser::final_parser, ParserExt};
use proptest::prelude::*;

type ParserResult<'a, T> = IResult<&'a str, T, ErrorTree<&'a str>>;

pub trait StringToCst {
    fn parse_cst(&self) -> Vec<Cst>;
}
impl StringToCst for str {
    fn parse_cst(&self) -> Vec<Cst> {
        // TODO: handle trailing whitespace and comments properly
        let source = format!("\n{}", self);
        let parser = map(
            tuple((|input| expressions0(&source, input, 0), many0(line_ending))),
            |(csts, _)| csts,
        );
        let result: Result<_, ErrorTree<&str>> = final_parser(parser)(&source);
        match result {
            Ok(parsed) => fix_offsets_csts(parsed),
            Err(err) => vec![Cst::Error {
                offset: 0,
                unparsable_input: self.to_owned(),
                message: format!("An error occurred while parsing: {:?}", err),
            }],
        }
    }
}

/// Because we don't parse the input directly, but prepend a newline to it, we
/// need to adjust the offsets of the CSTs to account for that.
fn fix_offsets_csts(csts: Vec<Cst>) -> Vec<Cst> {
    csts.into_iter().map(|cst| fix_offsets_cst(cst)).collect()
}
fn fix_offsets_cst(cst: Cst) -> Cst {
    match cst {
        Cst::EqualsSign { offset } => Cst::EqualsSign { offset: offset - 1 },
        Cst::OpeningParenthesis { offset } => Cst::OpeningParenthesis { offset: offset - 1 },
{'fraction_non_alphanumeric': 0.1523029104266742, 'fraction_numerical': 0.004803616840915513, 'mean_word_length': 3.204275534441805, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5996840', 'n_tokens_mistral': 1175, 'n_tokens_neox': 1172, 'n_words': 289}