<commit_before>use std::ops::Range;

use scanner::Scanner;

/// Scan for email address starting from the trigger character "@".
///
/// Based on RFC 6531, but also scans invalid IDN. Doesn't match IP address in domain part or
/// quoting in local part.
pub struct EmailScanner {
    pub domain_must_have_dot: bool,
}

impl Scanner for EmailScanner {
    fn scan(&self, s: &str, at: usize) -> Option<Range<usize>> {
        if let Some(start) = self.find_start(s, at) {
            if let Some(end) = self.find_end(s, at + 1) {
                return Some(Range {
                    start: start,
                    end: end,
                });
            }
        }
        return None;
    }
}

impl EmailScanner {
    // See "Local-part" in RFC 5321, plus extensions in RFC 6531
    fn find_start(&self, s: &str, at: usize) -> Option<usize> {
        let mut first = None;
        let mut atom_boundary = true;
        for (i, c) in s[0..at].char_indices().rev() {
            if Self::local_atom_allowed(c) {
                first = Some(i);
                atom_boundary = false;
            } else if c == '.' {
                if atom_boundary {
                    break;
                }
                atom_boundary = true;
            } else {
                break;
            }
        }
        first
    }

    // See "Domain" in RFC 5321, plus extension of "sub-domain" in RFC 6531
    fn find_end(&self, s: &str, start: usize) -> Option<usize> {
        let mut first_in_sub_domain = true;
        let mut can_end_sub_domain = false;
        let mut first_dot = None;
        let mut last = None;

        for (i, c) in s[start..].char_indices() {
            if first_in_sub_domain {
                if Self::sub_domain_allowed(c) {
                    last = Some(i);
                    first_in_sub_domain = false;
                    can_end_sub_domain = true;
                } else {
                    break;
                }
            } else {
                if c == '.' {
                    if !can_end_sub_domain {
                        break;
                    }
                    first_in_sub_domain = true;
                    if first_dot.is_none() {
                        first_dot = Some(i);
                    }
                } else if c == '-' {
                    can_end_sub_domain = false;
                } else if Self::sub_domain_allowed(c) {
                    last = Some(i);
                    can_end_sub_domain = true;
                } else {
                    break;
                }
            }
        }

        if let Some(last) = last {
            if !self.domain_must_have_dot || first_dot.map(|d| d < last).unwrap_or(false) {
                Some(start + last + 1)
            } else {
                None
            }
        } else {
            None
        }
    }

    // See "Atom" in RFC 5321, "atext" in RFC 5322
    fn local_atom_allowed(c: char) -> bool {
        match c {
            'a' ... 'z' | 'A' ... 'Z' | '0' ... '9' | '!' | '#' | '$' | '%' | '&' | '\'' | '*' |
            '+' | '-' | '/' | '=' | '?' | '^' | '_' | '`' | '{' | '|' | '}' | '~' => true,
            _ => c >= '\u{80}',<commit_msg>Pass slices to email scanner methods as well<commit_after>use std::ops::Range;

use scanner::Scanner;

/// Scan for email address starting from the trigger character "@".
///
/// Based on RFC 6531, but also scans invalid IDN. Doesn't match IP address in domain part or
/// quoting in local part.
pub struct EmailScanner {
    pub domain_must_have_dot: bool,
}

impl Scanner for EmailScanner {
    fn scan(&self, s: &str, at: usize) -> Option<Range<usize>> {
        if let Some(start) = self.find_start(&s[0..at]) {
            let after = at + 1;
            if let Some(end) = self.find_end(&s[after..]) {
                return Some(Range {
                    start: start,
                    end: after + end,
                });
            }
        }
        return None;
    }
}

impl EmailScanner {
    // See "Local-part" in RFC 5321, plus extensions in RFC 6531
    fn find_start(&self, s: &str) -> Option<usize> {
        let mut first = None;
        let mut atom_boundary = true;
        for (i, c) in s.char_indices().rev() {
            if Self::local_atom_allowed(c) {
                first = Some(i);
                atom_boundary = false;
            } else if c == '.' {
                if atom_boundary {
                    break;
                }
                atom_boundary = true;
            } else {
                break;
            }
        }
        first
    }

    // See "Domain" in RFC 5321, plus extension of "sub-domain" in RFC 6531
    fn find_end(&self, s: &str) -> Option<usize> {
        let mut first_in_sub_domain = true;
        let mut can_end_sub_domain = false;
        let mut first_dot = None;
        let mut last = None;

        for (i, c) in s.char_indices() {
            if first_in_sub_domain {
                if Self::sub_domain_allowed(c) {
                    last = Some(i);
                    first_in_sub_domain = false;
                    can_end_sub_domain = true;
                } else {
                    break;
                }
            } else {
                if c == '.' {
                    if !can_end_sub_domain {
                        break;
                    }
                    first_in_sub_domain = true;
                    if first_dot.is_none() {
                        first_dot = Some(i);
                    }
                } else if c == '-' {
                    can_end_sub_domain = false;
                } else if Self::sub_domain_allowed(c) {
                    last = Some(i);
                    can_end_sub_domain = true;
                } else {
                    break;
                }
            }
        }

        if let Some(last) = last {
            if !self.domain_must_have_dot || first_dot.map(|d| d < last).unwrap_or(false) {
                Some(last + 1)
            } else {
                None
            }
        } else {
            None
        }
    }

    // See "Atom" in RFC 5321, "atext" in RFC 5322
    fn local_atom_allowed(c: char) -> bool {
        match c {
            'a' ... 'z' | 'A' ... 'Z' | '0' ... '9' | '!' | '#' | '$' | '%' | '&' | '\'' | '*' |
            '+' | '-' | '/' | '=' | '?' | '^' | '_' | '`' | '{' | '|' | '}' | '~' => true,
            _ => c >= '\u{80}',
{'fraction_non_alphanumeric': 0.12763975155279503, 'fraction_numerical': 0.010869565217391304, 'mean_word_length': 1.1556224899598393, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6558894', 'n_tokens_mistral': 2037, 'n_tokens_neox': 1880, 'n_words': 477}