<commit_before>// ordered container

use std::collections::BinaryHeap;

#[derive(Clone)]
pub struct OrderedVec<T: Ord> {
    heap: BinaryHeap<T>,
    vec: Vec<T>,
    sorted: bool,
    index_min: usize,
}

impl<T> OrderedVec<T> where T: Ord {
    pub fn new() -> Self {
        OrderedVec {
            heap: BinaryHeap::new(),
            vec: Vec::new(),
            sorted: true,
            index_min: 0,
        }
    }

    pub fn push(&mut self, item: T) {
        if self.index_min > 0 && item > self.vec[self.index_min/2] {
            self.vec.push(item);
            self.sorted = false;
        } else {
            self.heap.push(item);
        }
    }

    pub fn get(&mut self, index: usize) -> Option<&T> {
        if !self.sorted {
            self.vec.sort_by(|a, b| b.cmp(a));
        }

        if index >= self.vec.len() + self.heap.len() {
            return None;
        }

        let mut len = self.vec.len();
        while len <= index {
            self.vec.push(self.heap.pop().unwrap());
            len += 1;
        }

        self.sorted = true;
        self.index_min = self.vec.len();
        return self.vec.get(index);
    }

    pub fn len(&self) -> usize {
        return self.vec.len() + self.heap.len();
    }

    pub fn clear(&mut self) {
        self.vec.clear();
        self.heap.clear();
        self.sorted = true;
        self.index_min = 0;
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn test_push_get() {<commit_msg>[orderedvec] give up on binaryheap<commit_after>// ordered container
// Normally, user will only care about the first several options. So we only keep several of them
// in order. Other items are kept unordered and are sorted on demand.

const ORDERED_SIZE: usize = 300;

#[derive(Clone)]
pub struct OrderedVec<T: Ord> {
    ordered: Vec<T>,
    unordered: Vec<T>,
    sorted: bool,
}

impl<T> OrderedVec<T> where T: Ord {
    pub fn new() -> Self {
        OrderedVec {
            ordered: Vec::new(),
            unordered: Vec::new(),
            sorted: false,
        }
    }

    fn ordered_insert(&mut self, item: T) {
        self.ordered.push(item);
        let mut pos = self.ordered.len() - 1;
        while pos > 0 && self.ordered[pos] > self.ordered[pos-1] {
            self.ordered.swap(pos, pos-1);
            pos -= 1;
        }
    }

    pub fn push(&mut self, item: T) {
        if self.ordered.len() < ORDERED_SIZE {
            self.ordered_insert(item);
            return;
        } 

        let smaller = if item < *self.ordered.last().unwrap() {
            item
        } else {
            self.ordered_insert(item);
            self.ordered.pop().unwrap()
        };

        self.unordered.push(smaller);
        self.sorted = false;
    }

    pub fn get(&mut self, index: usize) -> Option<&T> {
        if index < self.ordered.len() {
            return self.ordered.get(index);
        }

        if index >= self.ordered.len() + self.unordered.len() {
            return None;
        }

        if !self.sorted {
            self.unordered.sort_by(|a, b| b.cmp(a));
            self.sorted = true;
        }

        return self.unordered.get(index - self.ordered.len());
    }

    pub fn len(&self) -> usize {
        return self.ordered.len() + self.unordered.len();
    }

    pub fn clear(&mut self) {
        self.ordered.clear();
        self.unordered.clear();
        self.sorted = true;
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn test_push_get() {
{'fraction_non_alphanumeric': 0.1454183266932271, 'fraction_numerical': 0.003699487763232783, 'mean_word_length': 1.6996927803379416, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1410326', 'n_tokens_mistral': 1154, 'n_tokens_neox': 1141, 'n_words': 272}