<commit_before>//! Hyphenating iterators.

use std::borrow::Cow;

use unicode_segmentation::UnicodeSegmentation;

use language::{Corpus};
use utilia::{Interspersable, Intersperse};

pub trait Hyphenation<Hyphenator> where Hyphenator : Iterator {
    /// Returns the indices of valid hyphenation points within the text.
    fn opportunities(self, corp: &Corpus) -> Vec<usize>;

    /// Returns an iterator over segments of the text separated by valid
    /// hyphenation points.
    ///
    /// When iterating over a word, such segments coincide with orthographic
    /// syllables. Note that, in some orthographies, the syllables of a hyphenated
    /// word are not necessarily substrings of the original word.
    fn hyphenate(self, corp: &Corpus) -> Hyphenator;
}


/// The `Standard` hyphenator iterates over a string, returning slices
/// delimited by string boundaries and valid hyphenation points.
///
/// For individual words, such slices coincide with orthographic syllables.
#[derive(Clone, Debug)]
pub struct Standard<'a> {
    text: &'a str,
    opportunities: Vec<usize>,
    prior: usize,
    i: usize
}

impl<'a> Standard<'a> {
    /// Inserts a soft hyphen at hyphenation points.
    pub fn punctuate(self) -> Intersperse<Self> {
        self.intersperse("\u{ad}")
    }

    /// Inserts a given string at hyphenation points.
    pub fn punctuate_with(self, mark: &'a str) -> Intersperse<Self> {
        self.intersperse(mark)
    }
}


impl<'a> Iterator for Standard<'a> {
    type Item = &'a str;

    fn next(&mut self) -> Option<&'a str> {
        let start = self.prior;
        let i = self.i;

        match self.opportunities.get(i) {
            Some(&end) => {
                self.prior = end;
                self.i = i + 1;
                Some(&self.text[start .. end])
            },
            None => {
                if i <= self.opportunities.len() {
                    self.i = i + 1;
                    Some(&self.text[start ..])
                } else {
                    None
                }
            }
        }
    }
}


impl<'a> Hyphenation<Standard<'a>> for &'a str {
    /// Returns the byte indices of valid hyphenation points within the string.
    fn opportunities(self, corp: &Corpus) -> Vec<usize> {
        let (l_min, r_min) = (corp.left_min, corp.right_min);
        let length_min = l_min + r_min;

        if self.chars().count() < length_min {
            return vec![];
        }

        let by_word = self.split_word_bound_indices();

        by_word.flat_map(|(i, word)| {
            let pts = match corp.exceptions.iter()
                                .filter_map(|exs| exs.score(word))
                                .next() {
                    Some(vec) => Cow::Borrowed(vec),
                    None => Cow::Owned(corp.patterns.score(word))
            }.into_owned();
            let length = pts.len();
            let l = l_min;
            let r = if length >= length_min { length - l_min - r_min + 1 } else { 0 };

            word.char_indices().skip(l)
                .zip(pts.into_iter().skip(l).take(r))
                .filter(|&(_, p)| p % 2 != 0)
                .map(move |((i1, _), _)| i1 + i)
        }).collect()
    }

    /// Returns an iterator over string slices separated by valid hyphenation
    /// points.
    fn hyphenate(self, corp: &Corpus) -> Standard<'a> {
        let os = self.opportunities(corp);

        Standard {
            text: self,
            opportunities: os,
            prior: 0,
            i: 0
        }<commit_msg>Code trivialities<commit_after>//! Hyphenating iterators.

use std::borrow::Cow;

use unicode_segmentation::UnicodeSegmentation;

use language::{Corpus};
use utilia::{Interspersable, Intersperse};

pub trait Hyphenation<Hyphenator> where Hyphenator : Iterator {
    /// Returns the indices of valid hyphenation points within the text.
    fn opportunities(self, corp: &Corpus) -> Vec<usize>;

    /// Returns an iterator over segments of the text separated by valid
    /// hyphenation points.
    ///
    /// When iterating over a word, such segments coincide with orthographic
    /// syllables. Note that, in some orthographies, the syllables of a hyphenated
    /// word are not necessarily substrings of the original word.
    fn hyphenate(self, corp: &Corpus) -> Hyphenator;
}


/// The `Standard` hyphenator iterates over a string, returning slices
/// delimited by string boundaries and valid hyphenation points.
///
/// For individual words, such slices coincide with orthographic syllables.
#[derive(Clone, Debug)]
pub struct Standard<'a> {
    text: &'a str,
    opportunities: Vec<usize>,
    prior: usize,
    i: usize
}

impl<'a> Standard<'a> {
    /// Inserts a soft hyphen at hyphenation points.
    pub fn punctuate(self) -> Intersperse<Self> {
        self.intersperse("\u{ad}")
    }

    /// Inserts a given string at hyphenation points.
    pub fn punctuate_with(self, mark: &'a str) -> Intersperse<Self> {
        self.intersperse(mark)
    }
}


impl<'a> Iterator for Standard<'a> {
    type Item = &'a str;

    fn next(&mut self) -> Option<&'a str> {
        let start = self.prior;
        let i = self.i;

        match self.opportunities.get(i) {
            Some(&end) => {
                self.prior = end;
                self.i = i + 1;
                Some(&self.text[start .. end])
            },
            None => {
                if i <= self.opportunities.len() {
                    self.i = i + 1;
                    Some(&self.text[start ..])
                } else {
                    None
                }
            }
        }
    }
}


impl<'a> Hyphenation<Standard<'a>> for &'a str {
    /// Returns the byte indices of valid hyphenation points within the string.
    fn opportunities(self, corp: &Corpus) -> Vec<usize> {
        let (l_min, r_min) = (corp.left_min, corp.right_min);
        let length_min = l_min + r_min;

        if self.chars().count() < length_min {
            return vec![];
        }

        let by_word = self.split_word_bound_indices();

        by_word.flat_map(|(i, word)| {
            let pts = match corp.exceptions.iter()
                                .filter_map(|exs| exs.score(word))
                                .next() {
                    Some(vec) => Cow::Borrowed(vec),
                    None => Cow::Owned(corp.patterns.score(word))
            }.into_owned();
            let length = pts.len();
            let l = l_min;
            let r = if length >= length_min { length - l_min - r_min + 1 } else { 0 };

            word.char_indices().skip(l)
                .zip(pts.into_iter().skip(l).take(r))
                .filter(|&(_, p)| p % 2 != 0)
                .map(move |((i1, _), _)| i1 + i)
        }).collect()
    }

    /// Returns an iterator over string slices separated by valid hyphenation
    /// points.
    fn hyphenate(self, corp: &Corpus) -> Standard<'a> {
        Standard {
            text: self,
            opportunities: self.opportunities(corp),
            prior: 0,
            i: 0
        }
{'fraction_non_alphanumeric': 0.1272264631043257, 'fraction_numerical': 0.0028272547356516823, 'mean_word_length': 1.8311324529811925, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5271868', 'n_tokens_mistral': 2211, 'n_tokens_neox': 2108, 'n_words': 594}