<commit_before>use std::borrow::Borrow;
use std::cmp::Ordering;
use std::collections::BTreeMap;
use std::collections::btree_map::Entry::*;

use util::*;

/// Information about the position of a single term within a document
#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, RustcEncodable)]
pub struct Position {
    /// Pair of byte indexes into the document at the beginning (inclusive) and end (exclusive) of 
    /// the term.
    pub offsets: (usize, usize),
    /// The token position of the term, i.e., the number of tokens that occur before it in the doc.
    /// For example, for the sentence "I have to go to the store",
    /// the term "to" has positions [2, 4].
    pub position: usize,
}

impl Position {
    /// Creates a new Position struct with the given offsets and position.
    pub fn new(offsets: (usize, usize), position: usize) -> Position {
        Position {
            offsets: offsets,
            position: position,
        }
    }
}

impl Merge for Position {
    fn merge(self, other: Position) -> Option<Position> {
        if self.position == other.position {
            self.offsets.merge(other.offsets).map(|offsets| Position::new(offsets, self.position))
        } else {
            None
        }
    }
}

/// Each document is assigned a unique string id.
pub type DocId = String;

/// A Postings map (doc id => positions) for a single term.
/// Records which Documents contain the term, and at which locations in the documents.
pub type PostingsMap = BTreeMap<DocId, Vec<Position>>;

/// An extension trait for iterables over PostingsMaps
/// that enables the computation of the union of Postingsmaps
pub trait PostingsMerge {
    /// Computes the map containing the union of the maps in self
    fn merge_postings(self) -> PostingsMap;
}

impl<'a, Iter> PostingsMerge for Iter
    where Iter: IntoIterator,
          Iter::Item: Borrow<PostingsMap> {
    fn merge_postings(self) -> PostingsMap {
        let mut map = PostingsMap::new();
        for tree in self {
            for (doc_id, positions) in tree.borrow() {
                match map.entry(doc_id.clone()) {
                    Vacant(entry) => {
                        entry.insert(positions.clone());
                    }
                    Occupied(mut entry) =>
                        entry.get_mut().merge_coalesce(positions.iter().cloned()),
                }
            }
        }
        map
    }
}

/// An extension trait for slices of PostingsMaps,
/// that enables the computation of the intersection
/// of PostingsMaps<commit_msg>Clean up a few comments<commit_after>use std::borrow::Borrow;
use std::cmp::Ordering;
use std::collections::BTreeMap;
use std::collections::btree_map::Entry::*;

use util::*;

/// Information about the position of a single term within a document
#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, RustcEncodable)]
pub struct Position {
    /// Pair of byte indexes into the document at the beginning (inclusive) and end (exclusive) of 
    /// the term.
    pub offsets: (usize, usize),
    /// The token position of the term, i.e., the number of tokens that occur before it in the doc.
    /// For example, for the sentence "I have to go to the store",
    /// the term "to" has positions [2, 4].
    pub position: usize,
}

impl Position {
    /// Creates a new Position struct with the given offsets and position.
    pub fn new(offsets: (usize, usize), position: usize) -> Position {
        Position {
            offsets: offsets,
            position: position,
        }
    }
}

impl Merge for Position {
    fn merge(self, other: Position) -> Option<Position> {
        if self.position == other.position {
            self.offsets.merge(other.offsets).map(|offsets| Position::new(offsets, self.position))
        } else {
            None
        }
    }
}

/// Each document is assigned a unique string id.
pub type DocId = String;

/// A postings map (doc id => positions) for a single term.
/// Records which Documents contain the term, and at which locations in the documents.
pub type PostingsMap = BTreeMap<DocId, Vec<Position>>;

/// An extension trait for iterables over `PostingsMap`s
/// that enables computing their union.
pub trait PostingsMerge {
    /// Computes the map containing the union of the maps in self
    fn merge_postings(self) -> PostingsMap;
}

impl<'a, Iter> PostingsMerge for Iter
    where Iter: IntoIterator,
          Iter::Item: Borrow<PostingsMap> {
    fn merge_postings(self) -> PostingsMap {
        let mut map = PostingsMap::new();
        for tree in self {
            for (doc_id, positions) in tree.borrow() {
                match map.entry(doc_id.clone()) {
                    Vacant(entry) => {
                        entry.insert(positions.clone());
                    }
                    Occupied(mut entry) =>
                        entry.get_mut().merge_coalesce(positions.iter().cloned()),
                }
            }
        }
        map
    }
}

/// An extension trait for slices of `PostingsMap`s
/// that enables computing their intersection.
{'fraction_non_alphanumeric': 0.10873405299313052, 'fraction_numerical': 0.0007850834151128558, 'mean_word_length': 2.346027577150361, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6765598', 'n_tokens_mistral': 1439, 'n_tokens_neox': 1414, 'n_words': 513}