<commit_before>use DocId;
use postings::{Postings, DocSet};
use std::cmp::Ordering;
use std::collections::BinaryHeap;
use query::MultiTermScorer;
use postings::ScoredDocSet;
use query::Scorer;
use fastfield::U32FastFieldReader;


#[derive(Eq, PartialEq)]
struct HeapItem(DocId, usize, u32);

impl HeapItem {
    pub fn ord(&self,) -> usize{
        self.1
    }
}


impl PartialOrd for HeapItem {
    fn partial_cmp(&self, other:&Self) -> Option<Ordering> {
        Some(self.cmp(&other))
    }
}

impl Ord for HeapItem {
    fn cmp(&self, other:&Self) -> Ordering {
         (self.0, self.1).cmp(&(other.0, other.1)).reverse()
    }
}

pub struct UnionPostings<TPostings: Postings> {
    fieldnorms_readers: Vec<U32FastFieldReader>,
    postings: Vec<TPostings>,
    queue: BinaryHeap<HeapItem>,
    doc: DocId,
    scorer: MultiTermScorer
}

impl<TPostings: Postings> UnionPostings<TPostings> {
        
    pub fn new(fieldnorms_reader: Vec<U32FastFieldReader>, mut postings: Vec<TPostings>, multi_term_scorer: MultiTermScorer) -> UnionPostings<TPostings> {
        let num_postings = postings.len();
        assert_eq!(fieldnorms_reader.len(), num_postings);

        for posting in &mut postings {
            assert!(posting.next());
        }
        let heap_items: Vec<HeapItem> = postings.iter()
            .map(|posting| {
                (posting.doc(), posting.term_freq())
            })
            .enumerate()
            .map(|(ord, (doc, tf))| {
                HeapItem(doc, ord, tf)
            })
            .collect();
        
        UnionPostings {
            fieldnorms_readers: fieldnorms_reader,
            postings: postings,
            queue: BinaryHeap::from(heap_items),
            doc: 0,
            scorer: multi_term_scorer
        }
    }

    fn advance_head(&mut self,) {
        let ord = self.queue.peek().unwrap().ord();
        let cur_postings = &mut self.postings[ord];
        if cur_postings.next() {
            let doc = cur_postings.doc();
            let tf = cur_postings.term_freq();
            self.queue.replace(HeapItem(doc, ord, tf));
        }
        else {
            self.queue.pop();
        }
        
    }
    
    fn get_field_norm(&self, ord:usize, doc:DocId) -> u32 {
        self.fieldnorms_readers[ord].get(doc)
    }

}

impl<TPostings: Postings> DocSet for UnionPostings<TPostings> {
    
       
    fn next(&mut self,) -> bool {
        self.scorer.clear();
        let cur_doc: DocId;
        match self.queue.peek() {
            Some(&HeapItem(doc, ord, tf)) => {
                cur_doc = doc;
                let fieldnorm = self.get_field_norm(ord, doc);
                self.scorer.update(ord, tf, fieldnorm);
                
            }
            None => {
                return false;
            }
        }
        self.advance_head();
        loop {
            match self.queue.peek() {
                Some(&HeapItem(peek_doc, peek_ord, peek_tf))  => {
                    if peek_doc != cur_doc {
                        break;
                    }
                    else {
                        let fieldnorm = self.get_field_norm(peek_ord, peek_doc);
                        self.scorer.update(peek_ord, peek_tf, fieldnorm);
                    }
                }
                None => { break; }   
            }
            self.advance_head();
        }
        self.doc = cur_doc;
        return true;
    }

    // TODO implement a faster skip_next
        
    fn doc(&self,) -> DocId {
        self.doc
    }
        
    fn doc_freq(&self,) -> usize {
        panic!("Doc freq");
    }
}

impl<TPostings: Postings> ScoredDocSet for UnionPostings<TPostings> {
    fn score(&self,) -> f32 {<commit_msg>Lighter HeapItem by putting term freq aside, and keeping ord as u32<commit_after>use DocId;
use postings::{Postings, DocSet};
use std::cmp::Ordering;
use std::collections::BinaryHeap;
use query::MultiTermScorer;
use postings::ScoredDocSet;
use query::Scorer;
use fastfield::U32FastFieldReader;
use std::iter;

#[derive(Eq, PartialEq)]
struct HeapItem(DocId, u32);

impl HeapItem {
    pub fn ord(&self,) -> usize{
        self.1 as usize
    }
}

impl PartialOrd for HeapItem {
    fn partial_cmp(&self, other:&Self) -> Option<Ordering> {
        Some(self.cmp(&other))
    }
}

impl Ord for HeapItem {
    fn cmp(&self, other:&Self) -> Ordering {
         (other.0).cmp(&self.0)
    }
}

pub struct UnionPostings<TPostings: Postings> {
    fieldnorms_readers: Vec<U32FastFieldReader>,
    postings: Vec<TPostings>,
    term_frequencies: Vec<u32>,
    queue: BinaryHeap<HeapItem>,
    doc: DocId,
    scorer: MultiTermScorer
}

impl<TPostings: Postings> UnionPostings<TPostings> {
        
    pub fn new(fieldnorms_reader: Vec<U32FastFieldReader>, mut postings: Vec<TPostings>, multi_term_scorer: MultiTermScorer) -> UnionPostings<TPostings> {
        let num_postings = postings.len();
        assert_eq!(fieldnorms_reader.len(), num_postings);
        
        for posting in &mut postings {
            assert!(posting.next());
        }
        let mut term_frequencies: Vec<u32> = iter::repeat(0u32).take(num_postings).collect();
        let heap_items: Vec<HeapItem> = postings
            .iter()
            .map(|posting| {
                (posting.doc(), posting.term_freq())
            })
            .enumerate()
            .map(|(ord, (doc, tf))| {
                term_frequencies[ord] = tf;
                HeapItem(doc, ord as u32)
            })
            .collect();
        
        UnionPostings {
            fieldnorms_readers: fieldnorms_reader,
            postings: postings,
            term_frequencies: term_frequencies,
            queue: BinaryHeap::from(heap_items),
            doc: 0,
            scorer: multi_term_scorer
        }
    }

    fn advance_head(&mut self,) {
        let ord = self.queue.peek().unwrap().ord();
        let cur_postings = &mut self.postings[ord];
        if cur_postings.next() {
            let doc = cur_postings.doc();
            self.term_frequencies[ord] = cur_postings.term_freq();  
            self.queue.replace(HeapItem(doc, ord as u32));
        }
        else {
            self.queue.pop();
        }
    }
    
    fn get_field_norm(&self, ord:usize, doc:DocId) -> u32 {
        self.fieldnorms_readers[ord].get(doc)
    }

}

impl<TPostings: Postings> DocSet for UnionPostings<TPostings> {
    
       
    fn next(&mut self,) -> bool {
        self.scorer.clear();
        let cur_doc: DocId;
        match self.queue.peek() {
            Some(&HeapItem(doc, ord)) => {
                cur_doc = doc;
                let ord: usize = ord as usize;
                let fieldnorm = self.get_field_norm(ord, doc);
                let tf = self.term_frequencies[ord];
                self.scorer.update(ord, tf, fieldnorm);
                
            }
            None => {
                return false;
            }
        }
        self.advance_head();
        loop {
            match self.queue.peek() {
                Some(&HeapItem(peek_doc, peek_ord))  => {
                    if peek_doc != cur_doc {
                        break;
                    }
                    else {
                        let peek_ord: usize = peek_ord as usize;
                        let peek_tf = self.term_frequencies[peek_ord];
                        let fieldnorm = self.get_field_norm(peek_ord, peek_doc);
                        self.scorer.update(peek_ord, peek_tf, fieldnorm);
                    }
                }
                None => { break; }   
            }
            self.advance_head();
        }
        self.doc = cur_doc;
        return true;
    }

    // TODO implement a faster skip_next
        
    fn doc(&self,) -> DocId {
        self.doc
    }
        
    fn doc_freq(&self,) -> usize {
        panic!("Doc freq");
    }
}

impl<TPostings: Postings> ScoredDocSet for UnionPostings<TPostings> {
    fn score(&self,) -> f32 {
{'fraction_non_alphanumeric': 0.11902071563088512, 'fraction_numerical': 0.005900816070307595, 'mean_word_length': 1.745949672526715, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2951369', 'n_tokens_mistral': 2556, 'n_tokens_neox': 2497, 'n_words': 483}