<commit_before>use byteorder::{ByteOrder, BigEndian};
use rocksdb;

use {RocksDBIndexStore, RocksDBIndexReader};
use segment::Segment;
use key_builder::KeyBuilder;


#[derive(Debug)]
pub struct SegmentStatistics {
    total_docs: i64,
    deleted_docs: i64,
}


impl SegmentStatistics {
    fn read(reader: &RocksDBIndexReader, segment: u32) -> Result<SegmentStatistics, rocksdb::Error> {
        // Total docs
        let kb = KeyBuilder::segment_stat(segment, b"total_docs");

        let total_docs = match try!(reader.snapshot.get(&kb.key())) {
            Some(val_bytes) => {
                BigEndian::read_i64(&val_bytes)
            }
            None => 0,
        };

        // Deleted docs
        let kb = KeyBuilder::segment_stat(segment, b"deleted_docs");

        let deleted_docs = match try!(reader.snapshot.get(&kb.key())) {
            Some(val_bytes) => {
                BigEndian::read_i64(&val_bytes)
            }
            None => 0,
        };

        Ok(SegmentStatistics {
            total_docs: total_docs,
            deleted_docs: deleted_docs,
        })
    }

    #[inline]
    pub fn total_docs(&self) -> i64 {
        self.total_docs
    }

    #[inline]
    pub fn deleted_docs(&self) -> i64 {
        self.deleted_docs
    }
}


impl RocksDBIndexStore {
    pub fn get_segment_statistics(&self) -> Result<Vec<(u32, SegmentStatistics)>, rocksdb::Error> {
        let mut segment_stats = Vec::new();
        let reader = self.reader();

        for segment in self.segments.iter_active(&reader) {
            let stats = try!(SegmentStatistics::read(&reader, segment.id()));
            segment_stats.push((segment.id(), stats));
        }

        Ok(segment_stats)
    }
}
<commit_msg>Make SegmentStatistics generic<commit_after>use byteorder::{ByteOrder, BigEndian};
use rocksdb;

use {RocksDBIndexStore, RocksDBIndexReader};
use segment::Segment;
use key_builder::KeyBuilder;


#[derive(Debug)]
pub struct SegmentStatistics {
    total_docs: i64,
    deleted_docs: i64,
}


impl SegmentStatistics {
    fn read<S: Segment>(reader: &RocksDBIndexReader, segment: &S) -> Result<SegmentStatistics, rocksdb::Error> {
        let total_docs = try!(segment.load_statistic(b"total_docs")).unwrap_or(0);
        let deleted_docs = try!(segment.load_statistic(b"deleted_docs")).unwrap_or(0);

        Ok(SegmentStatistics {
            total_docs: total_docs,
            deleted_docs: deleted_docs,
        })
    }

    #[inline]
    pub fn total_docs(&self) -> i64 {
        self.total_docs
    }

    #[inline]
    pub fn deleted_docs(&self) -> i64 {
        self.deleted_docs
    }
}


impl RocksDBIndexStore {
    pub fn get_segment_statistics(&self) -> Result<Vec<(u32, SegmentStatistics)>, rocksdb::Error> {
        let mut segment_stats = Vec::new();
        let reader = self.reader();

        for segment in self.segments.iter_active(&reader) {
            let stats = try!(SegmentStatistics::read(&reader, &segment));
            segment_stats.push((segment.id(), stats));
        }

        Ok(segment_stats)
    }
}

{'fraction_non_alphanumeric': 0.12716009129442452, 'fraction_numerical': 0.0097815454841865, 'mean_word_length': 2.6698564593301435, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7370456', 'n_tokens_mistral': 972, 'n_tokens_neox': 953, 'n_words': 181}