<commit_before>use super::namepart::{Location, NamePart};
use super::suffix;
use super::surname;
use super::title;
use super::utils::is_mixed_case;
use smallvec::SmallVec;

struct ParseOp<'a> {
    words: SmallVec<[NamePart<'a>; 7]>,
    surname_index: usize,
    generation_from_suffix: Option<usize>,
    possible_false_prefix: Option<NamePart<'a>>,
    possible_false_postfix: Option<NamePart<'a>>,
    use_capitalization: bool,
}

pub fn parse(name: &str) -> Option<(SmallVec<[NamePart; 7]>, usize, Option<usize>)> {
    let mut op = ParseOp {
        words: SmallVec::new(),
        surname_index: 0,
        generation_from_suffix: None,
        possible_false_prefix: None,
        possible_false_postfix: None,
        use_capitalization: is_mixed_case(name),
    };

    if op.run(name) {
        Some((op.words, op.surname_index, op.generation_from_suffix))
    } else {
        None
    }
}

impl<'a> ParseOp<'a> {
    fn run(&mut self, name: &'a str) -> bool {
        // Separate comma-separated titles and suffixes, then flip remaining words
        // around remaining comma, if any
        if name.contains(',') {
            for part in name.split(',') {
                if self.words.is_empty() {
                    // We're in the surname part (if the format is "Smith, John"),
                    // or the only actual name part (if the format is "<NAME>,
                    // esq." or just "<NAME>")
                    self.handle_before_comma(part, true);
                } else if self.surname_index == 0 {
                    // We already processed one comma-separated part, but we think
                    // it was just the surname, so this might be the given name or
                    // initials
                    self.handle_after_comma(part);
                } else {
                    // We already found the full name, so this is a comma-separated
                    // postfix title or suffix
                    self.handle_after_surname(part);
                }
            }
        } else {
            self.handle_before_comma(name, false);
        }

        // If there are two or fewer words, e.g. "<NAME>", we treat
        // ambiguous strings like "MA" as surnames rather than titles
        // (not initials, which should only be at the end of the input
        // if they are comma-separated, and we already handled that case)
        if !self.valid() {
            if let Some(ref postfix) = self.possible_false_postfix {
                self.words.push(postfix.clone());
            } else if let Some(ref prefix) = self.possible_false_prefix {
                self.words.insert(0, prefix.clone());
            }
        }

        // Anything trailing that looks like initials is probably a stray postfix
        while self.words.last().filter(|w| !w.is_namelike()).is_some() {
            let removed = self.words.pop().unwrap();

            // If we guessed the surname previously, our guess is no longer valid
            self.surname_index = 0;

            // If the alternative is having less than two words and giving up,
            // check if we can treat the last word as a name if we just ignore
            // case; this handles the not-quite-rare-enough case of an all-caps
            // last name (e.g.<NAME>), among others
            if self.use_capitalization && !self.valid() {
                let word = NamePart::from_word(removed.word, false, Location::End);
                if word.is_namelike() {
                    self.words.push(word);
                    break;
                }
            }
        }

        // Handle case where we thought the whole before-comma part was a surname,
        // but we never found a plausible given name or initial afterwards,
        // as well as the reset just above
        if self.surname_index == 0 && self.words.len() > 1 {
            self.surname_index = surname::find_surname_index(&self.words[1..]) + 1;
        }

        debug_assert!(self
            .words
            .iter()
            .all(|w| w.is_namelike() || w.is_initials()));

        // Check the plausibility of what we've found
        self.valid()
    }

    fn valid(&self) -> bool {
        self.words.len() >= 2
            && self.words[self.surname_index..]
                .iter()
                .any(|w| w.is_namelike())
    }

    // Called only until any words are found
    fn handle_before_comma(&mut self, part: &'a str, any_after: bool) {
        debug_assert!(
            self.words.is_empty()
                && self.surname_index == 0
                && self.possible_false_prefix.is_none(),
            "Invalid state for handle_before_comma!"
        );

        self.words.extend(NamePart::all_from_text(
            part,
            self.use_capitalization,
            if any_after {
                Location::End
            } else {
                Location::Start
            },
        ));

        if self.words.is_empty() {
            return;
        }

        // Check for title as prefix (e.g. "Dr. <NAME>" or "Right Hon.
        // <NAME>")
        let prefix_title_len = title::find_prefix_len(&self.words);
        for i in (0..prefix_title_len).rev() {
            let word = self.words.remove(i);
            self.found_prefix(word);
        }

        // Strip non-comma-separated titles & suffixes (e.g. "<NAME> Jr.")
        let first_postfix_index = title::find_postfix_index(&self.words[1..], false) + 1;<commit_msg>More fast pathing<commit_after>use super::namepart::{Location, NamePart};
use super::suffix;
use super::surname;
use super::title;
use super::utils::is_mixed_case;
use smallvec::SmallVec;

struct ParseOp<'a> {
    words: SmallVec<[NamePart<'a>; 7]>,
    surname_index: usize,
    generation_from_suffix: Option<usize>,
    possible_false_prefix: Option<NamePart<'a>>,
    possible_false_postfix: Option<NamePart<'a>>,
    use_capitalization: bool,
}

pub fn parse(name: &str) -> Option<(SmallVec<[NamePart; 7]>, usize, Option<usize>)> {
    let mut op = ParseOp {
        words: SmallVec::new(),
        surname_index: 0,
        generation_from_suffix: None,
        possible_false_prefix: None,
        possible_false_postfix: None,
        use_capitalization: is_mixed_case(name),
    };

    if op.run(name) {
        Some((op.words, op.surname_index, op.generation_from_suffix))
    } else {
        None
    }
}

impl<'a> ParseOp<'a> {
    fn run(&mut self, name: &'a str) -> bool {
        // Separate comma-separated titles and suffixes, then flip remaining words
        // around remaining comma, if any
        if name.contains(',') {
            for part in name.split(',') {
                if self.words.is_empty() {
                    // We're in the surname part (if the format is "<NAME>"),
                    // or the only actual name part (if the format is "<NAME>,
                    // esq." or just "<NAME>")
                    self.handle_before_comma(part, true);
                } else if self.surname_index == 0 {
                    // We already processed one comma-separated part, but we think
                    // it was just the surname, so this might be the given name or
                    // initials
                    self.handle_after_comma(part);
                } else {
                    // We already found the full name, so this is a comma-separated
                    // postfix title or suffix
                    self.handle_after_surname(part);
                }
            }
        } else {
            self.handle_before_comma(name, false);
        }

        // If there are two or fewer words, e.g. "<NAME>", we treat
        // ambiguous strings like "MA" as surnames rather than titles
        // (not initials, which should only be at the end of the input
        // if they are comma-separated, and we already handled that case)
        if !self.valid() {
            if let Some(ref postfix) = self.possible_false_postfix {
                self.words.push(postfix.clone());
            } else if let Some(ref prefix) = self.possible_false_prefix {
                self.words.insert(0, prefix.clone());
            }
        }

        // Anything trailing that looks like initials is probably a stray postfix
        while self.words.last().filter(|w| !w.is_namelike()).is_some() {
            let removed = self.words.pop().unwrap();

            // If we guessed the surname previously, our guess is no longer valid
            self.surname_index = 0;

            // If the alternative is having less than two words and giving up,
            // check if we can treat the last word as a name if we just ignore
            // case; this handles the not-quite-rare-enough case of an all-caps
            // last name (e.g.<NAME>MITH), among others
            if self.use_capitalization && !self.valid() {
                let word = NamePart::from_word(removed.word, false, Location::End);
                if word.is_namelike() {
                    self.words.push(word);
                    break;
                }
            }
        }

        // Handle case where we thought the whole before-comma part was a surname,
        // but we never found a plausible given name or initial afterwards,
        // as well as the reset just above
        if self.surname_index == 0 && self.words.len() > 1 {
            self.surname_index = surname::find_surname_index(&self.words[1..]) + 1;
        }

        // Check the plausibility of what we've found
        self.valid()
    }

    fn valid(&self) -> bool {
        self.words.len() >= 2
            && self
                .words
                .iter()
                .all(|w| w.is_namelike() || w.is_initials())
            && self.words[self.surname_index..]
                .iter()
                .any(|w| w.is_namelike())
    }

    // Called only until any words are found
    fn handle_before_comma(&mut self, part: &'a str, any_after: bool) {
        debug_assert!(
            self.words.is_empty()
                && self.surname_index == 0
                && self.possible_false_prefix.is_none(),
            "Invalid state for handle_before_comma!"
        );

        self.words.extend(NamePart::all_from_text(
            part,
            self.use_capitalization,
            if any_after {
                Location::End
            } else {
                Location::Start
            },
        ));

        if self.words.is_empty() {
            return;
        }

        // Check for title as prefix (e.g. "Dr. <NAME>" or "Right Hon.
        // <NAME>")
        let prefix_title_len = if any_after || self.words.len() > 2 {
            title::find_prefix_len(&self.words)
        } else {
            0
        };
        for i in (0..prefix_title_len).rev() {
            let word = self.words.remove(i);
            self.found_prefix(word);
        }

        // Strip non-comma-separated titles & suffixes (e.g. "<NAME> Jr.")
        let first_postfix_index =
            if any_after || self.words.len() + self.possible_false_prefix.iter().count() > 2 {
                title::find_postfix_index(&self.words[1..], false) + 1
            } else {
                self.words.len()
            };
{'fraction_non_alphanumeric': 0.11227037631532014, 'fraction_numerical': 0.0029427501337613697, 'mean_word_length': 1.691384689224862, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 56, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3457756', 'n_tokens_mistral': 3298, 'n_tokens_neox': 3107, 'n_words': 891}