<commit_before>use byteorder::ByteOrder;
use common::bitpacker::BitPacker;
use common::compute_num_bits;
use common::Endianness;
use common::{BinarySerializable, FixedSize};
use directory::ReadOnlySource;
use postings::TermInfo;
use std::cmp;
use std::io;
use std::io::{Read, Write};
use termdict::TermOrdinal;

const BLOCK_LEN: usize = 256;

#[derive(Debug, Eq, PartialEq, Default)]
struct TermInfoBlockMeta {
    offset: u64,
    ref_term_info: TermInfo,
    doc_freq_nbits: u8,
    postings_offset_nbits: u8,
    positions_offset_nbits: u8,
}

impl BinarySerializable for TermInfoBlockMeta {
    fn serialize<W: Write>(&self, write: &mut W) -> io::Result<()> {
        self.offset.serialize(write)?;
        self.ref_term_info.serialize(write)?;
        write.write_all(&[
            self.doc_freq_nbits,
            self.postings_offset_nbits,
            self.positions_offset_nbits,
        ])?;
        Ok(())
    }

    fn deserialize<R: Read>(reader: &mut R) -> io::Result<Self> {
        let offset = u64::deserialize(reader)?;
        let ref_term_info = TermInfo::deserialize(reader)?;
        let mut buffer = [0u8; 3];
        reader.read_exact(&mut buffer)?;
        Ok(TermInfoBlockMeta {
            offset,
            ref_term_info,
            doc_freq_nbits: buffer[0],
            postings_offset_nbits: buffer[1],
            positions_offset_nbits: buffer[2],
        })
    }
}

impl FixedSize for TermInfoBlockMeta {
    const SIZE_IN_BYTES: usize =
        u64::SIZE_IN_BYTES + TermInfo::SIZE_IN_BYTES + 3 * u8::SIZE_IN_BYTES;
}

impl TermInfoBlockMeta {
    fn num_bits(&self) -> u8 {
        self.doc_freq_nbits + self.postings_offset_nbits + self.positions_offset_nbits + 7
    }

    fn deserialize_term_info(&self, data: &[u8], inner_offset: usize) -> TermInfo {
        let num_bits = self.num_bits() as usize;
        let mut cursor = num_bits * inner_offset;

        let doc_freq = extract_bits(data, cursor, self.doc_freq_nbits) as u32;
        cursor += self.doc_freq_nbits as usize;

        let postings_offset = extract_bits(data, cursor, self.postings_offset_nbits);
        cursor += self.postings_offset_nbits as usize;

        let positions_offset = extract_bits(data, cursor, self.positions_offset_nbits);
        cursor += self.positions_offset_nbits as usize;

        let positions_inner_offset = extract_bits(data, cursor, 7) as u8;

        TermInfo {
            doc_freq,
            postings_offset: postings_offset + self.ref_term_info.postings_offset,
            positions_offset: positions_offset + self.ref_term_info.positions_offset,
            positions_inner_offset,
        }
    }
}

pub struct TermInfoStore {
    num_terms: usize,
    block_meta_source: ReadOnlySource,
    term_info_source: ReadOnlySource,
}

fn extract_bits(data: &[u8], addr_bits: usize, num_bits: u8) -> u64 {
    assert!(num_bits <= 56);
    let addr_byte = addr_bits / 8;
    let bit_shift = (addr_bits % 8) as u64;
    let val_unshifted_unmasked: u64 = unsafe { *(data[addr_byte..].as_ptr() as *const u64) };
    let val_shifted_unmasked = val_unshifted_unmasked >> bit_shift;
    let mask = (1u64 << u64::from(num_bits)) - 1;
    val_shifted_unmasked & mask
}

impl TermInfoStore {
    pub fn open(data: &ReadOnlySource) -> TermInfoStore {
        let buffer = data.as_slice();
        let len = Endianness::read_u64(&buffer[0..8]) as usize;
        let num_terms = Endianness::read_u64(&buffer[8..16]) as usize;
        let block_meta_source = data.slice(16, 16 + len);
        let term_info_source = data.slice_from(16 + len);
        TermInfoStore {
            num_terms,
            block_meta_source,
            term_info_source,
        }
    }

    pub fn get(&self, term_ord: TermOrdinal) -> TermInfo {<commit_msg>Removed ptr dereference for explicit ptr::read_unaligned<commit_after>use byteorder::ByteOrder;
use common::bitpacker::BitPacker;
use common::compute_num_bits;
use common::Endianness;
use common::{BinarySerializable, FixedSize};
use directory::ReadOnlySource;
use postings::TermInfo;
use std::cmp;
use std::ptr;
use std::io::{self, Read, Write};
use termdict::TermOrdinal;

const BLOCK_LEN: usize = 256;

#[derive(Debug, Eq, PartialEq, Default)]
struct TermInfoBlockMeta {
    offset: u64,
    ref_term_info: TermInfo,
    doc_freq_nbits: u8,
    postings_offset_nbits: u8,
    positions_offset_nbits: u8,
}

impl BinarySerializable for TermInfoBlockMeta {
    fn serialize<W: Write>(&self, write: &mut W) -> io::Result<()> {
        self.offset.serialize(write)?;
        self.ref_term_info.serialize(write)?;
        write.write_all(&[
            self.doc_freq_nbits,
            self.postings_offset_nbits,
            self.positions_offset_nbits,
        ])?;
        Ok(())
    }

    fn deserialize<R: Read>(reader: &mut R) -> io::Result<Self> {
        let offset = u64::deserialize(reader)?;
        let ref_term_info = TermInfo::deserialize(reader)?;
        let mut buffer = [0u8; 3];
        reader.read_exact(&mut buffer)?;
        Ok(TermInfoBlockMeta {
            offset,
            ref_term_info,
            doc_freq_nbits: buffer[0],
            postings_offset_nbits: buffer[1],
            positions_offset_nbits: buffer[2],
        })
    }
}

impl FixedSize for TermInfoBlockMeta {
    const SIZE_IN_BYTES: usize =
        u64::SIZE_IN_BYTES + TermInfo::SIZE_IN_BYTES + 3 * u8::SIZE_IN_BYTES;
}

impl TermInfoBlockMeta {
    fn num_bits(&self) -> u8 {
        self.doc_freq_nbits + self.postings_offset_nbits + self.positions_offset_nbits + 7
    }

    fn deserialize_term_info(&self, data: &[u8], inner_offset: usize) -> TermInfo {
        let num_bits = self.num_bits() as usize;
        let mut cursor = num_bits * inner_offset;

        let doc_freq = extract_bits(data, cursor, self.doc_freq_nbits) as u32;
        cursor += self.doc_freq_nbits as usize;

        let postings_offset = extract_bits(data, cursor, self.postings_offset_nbits);
        cursor += self.postings_offset_nbits as usize;

        let positions_offset = extract_bits(data, cursor, self.positions_offset_nbits);
        cursor += self.positions_offset_nbits as usize;

        let positions_inner_offset = extract_bits(data, cursor, 7) as u8;

        TermInfo {
            doc_freq,
            postings_offset: postings_offset + self.ref_term_info.postings_offset,
            positions_offset: positions_offset + self.ref_term_info.positions_offset,
            positions_inner_offset,
        }
    }
}

pub struct TermInfoStore {
    num_terms: usize,
    block_meta_source: ReadOnlySource,
    term_info_source: ReadOnlySource,
}

fn extract_bits(data: &[u8], addr_bits: usize, num_bits: u8) -> u64 {
    assert!(num_bits <= 56);
    let addr_byte = addr_bits / 8;
    let bit_shift = (addr_bits % 8) as u64;
    let val_unshifted_unmasked: u64 = unsafe {
        let addr = data.as_ptr().offset(addr_byte as isize) as *const u64;
        ptr::read_unaligned(addr)
    };
    let val_shifted_unmasked = val_unshifted_unmasked >> bit_shift;
    let mask = (1u64 << u64::from(num_bits)) - 1;
    val_shifted_unmasked & mask
}

impl TermInfoStore {
    pub fn open(data: &ReadOnlySource) -> TermInfoStore {
        let buffer = data.as_slice();
        let len = Endianness::read_u64(&buffer[0..8]) as usize;
        let num_terms = Endianness::read_u64(&buffer[8..16]) as usize;
        let block_meta_source = data.slice(16, 16 + len);
        let term_info_source = data.slice_from(16 + len);
        TermInfoStore {
            num_terms,
            block_meta_source,
            term_info_source,
        }
    }

    pub fn get(&self, term_ord: TermOrdinal) -> TermInfo {
{'fraction_non_alphanumeric': 0.10829523187459177, 'fraction_numerical': 0.01619856303069889, 'mean_word_length': 3.0294736842105263, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3928972', 'n_tokens_mistral': 2610, 'n_tokens_neox': 2557, 'n_words': 529}