<commit_before>use std::mem;
use std::os::unix::prelude::AsRawFd;
use std::slice;
use std::ops;
use std::fs::File;
use std::marker::PhantomData;

use mmap;

pub struct TypedMemoryMap<T:Copy> {
    map:    mmap::MemoryMap,    // mapped file
    len:    usize,              // in bytes (needed because map extends to full block)
    phn:    PhantomData<T>,
}

impl<T:Copy> TypedMemoryMap<T> {
    pub fn new(filename: String) -> TypedMemoryMap<T> {
        let file = File::open(filename).ok().expect("error opening file");
        let size = file.metadata().ok().expect("error reading metadata").len() as usize;
        TypedMemoryMap {
            map: mmap::MemoryMap::new(size, &[mmap::MapOption::MapReadable,
                                              mmap::MapOption::MapFd(file.as_raw_fd())]).ok().expect("error mapping memory"),
            len: size / mem::size_of::<T>(),
            phn: PhantomData,
        }
    }
}

impl<T:Copy> ops::Index<ops::RangeFull> for TypedMemoryMap<T> {
    type Output = [T];
    #[inline]
    fn index(&self, _index: ops::RangeFull) -> &[T] {
        unsafe { slice::from_raw_parts(self.map.data() as *const T, self.len) }
    }
}<commit_msg>better error information<commit_after>use std::mem;
use std::os::unix::prelude::AsRawFd;
use std::slice;
use std::ops;
use std::fs::File;
use std::marker::PhantomData;

use mmap;

pub struct TypedMemoryMap<T:Copy> {
    map:    mmap::MemoryMap,    // mapped file
    len:    usize,              // in bytes (needed because map extends to full block)
    phn:    PhantomData<T>,
}

impl<T:Copy> TypedMemoryMap<T> {
    pub fn new(filename: String) -> TypedMemoryMap<T> {
        println!("attempting to open {}", filename);
        let file = File::open(filename).ok().expect("error opening file");
        let size = file.metadata().ok().expect("error reading metadata").len() as usize;
        TypedMemoryMap {
            map: mmap::MemoryMap::new(size, &[mmap::MapOption::MapReadable,
                                              mmap::MapOption::MapFd(file.as_raw_fd())]).ok().expect("error mapping memory"),
            len: size / mem::size_of::<T>(),
            phn: PhantomData,
        }
    }
}

impl<T:Copy> ops::Index<ops::RangeFull> for TypedMemoryMap<T> {
    type Output = [T];
    #[inline]
    fn index(&self, _index: ops::RangeFull) -> &[T] {
        unsafe { slice::from_raw_parts(self.map.data() as *const T, self.len) }
    }
}
{'fraction_non_alphanumeric': 0.16379310344827586, 'fraction_numerical': 0.0, 'mean_word_length': 2.8197492163009406, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1826367', 'n_tokens_mistral': 809, 'n_tokens_neox': 806, 'n_words': 174}