<commit_before>

#[cfg(windows)]
mod mmap {
    use libc::{CreateFileW, CreateFileMappingW, MapViewOfFile, CloseHandle, UnmapViewOfFile,
               GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING, INVALID_HANDLE_VALUE, PAGE_READONLY,
               FILE_MAP_READ, HANDLE, TRUE};
    use libc::consts::os::extra::{FILE_FLAG_RANDOM_ACCESS};
    use libc::types::os::arch::extra::{LPCVOID};
    use std::ptr;
    struct Handle {
        hand: HANDLE,
    }
    impl Drop for Handle {
        fn drop(&mut self) {
            assert_eq!(unsafe { CloseHandle(self.hand) }, TRUE);
        }
    }
    struct MapView {
        data: LPCVOID,
    }
    impl Drop for MapView {
        fn drop(&mut self) {
            assert_eq!(unsafe { UnmapViewOfFile(self.data) }, TRUE);
        }
    }
    pub struct MapFile {
        view: MapView,
        map: Handle,
        file: Handle,
    }
    impl MapFile {
        pub fn data(&self) -> LPCVOID { self.view.data }
    }
    fn create_file(path: &Path) -> Result<Handle, &'static str> {
        let name = path.as_str().unwrap().to_utf16().append([0]);
        let file = unsafe {
            CreateFileW(name.as_ptr(), GENERIC_READ, FILE_SHARE_READ, ptr::mut_null(),
                        OPEN_EXISTING, FILE_FLAG_RANDOM_ACCESS, ptr::mut_null())
        };
        if file.to_uint() == INVALID_HANDLE_VALUE as uint { return Err("Failed to open file"); }
        Ok(Handle{hand: file})
    }
    fn create_mapping(file: &Handle) -> Result<Handle, &'static str> {
        let map = unsafe {
            CreateFileMappingW(file.hand, ptr::mut_null(), PAGE_READONLY, 0, 0, ptr::null())
        };
        if map.is_null() { return Err("Failed to create file mapping"); }
        Ok(Handle{hand: map})
    }
    fn map_view(map: &Handle) -> Result<MapView, &'static str> {
        let view = unsafe { MapViewOfFile(map.hand, FILE_MAP_READ, 0, 0, 0) };
        if view.is_null() { return Err("Failed to map view of file"); }
        Ok(MapView{data: view as *_})
    }
    pub fn open(path: &Path) -> Result<MapFile, &'static str> {
        let file = try!(create_file(path));
        let map = try!(create_mapping(&file));
        let view = try!(map_view(&map));
        Ok(MapFile{file: file, map: map, view: view})
    }
}
#[cfg(unix)]
mod mmap {
    use libc::{close, fstat, mmap, munmap, O_RDONLY, PROT_READ, stat, c_int, c_void};
    use libc::consts::os::posix88::{MAP_SHARED};
    use posix_open = libc::open;
    use std::mem::{uninitialized, transmute};
    use std::ptr;
    use std::io::fs::stat;
    struct Handle {
        hand: c_int,
    }
    impl Drop for Handle {
        fn drop(&mut self) {
            assert_eq!(unsafe { close(self.hand) }, 0);
        }
    }
    struct Mapping {
        data: *mut c_void,
        size: u64,
    }
    impl Drop for Mapping {
        fn drop(&mut self) {
            assert_eq!(unsafe { munmap(self.data as *c_void, self.size) }, 0);
        }
    }
    struct MapFile {
        map: Mapping,
        file: Handle,
    }
    impl MapFile {
        pub fn data(&self) -> * c_void { self.map.data as *_ }
    }
    fn open_file(path: &Path) -> Result<Handle, &'static str> {
        let name = path.to_c_str();
        let handle = posix_open(name.unwrap(), O_RDONLY, 0);
        if (handle == -1) { return Err("Failed to open file") }
        Ok(Handle{hand: handle})
    }
    fn file_size(path: &Path) -> Result<u64, &'static str> {
        match path.stat() {
            Ok(stat) => Ok(stat.size),
            Err(_) => Err("Failed to get file size")
        }
    }
    fn map_file(file: &Handle, size: u64) -> Result<Mapping, &'static str> {
        let map = mmap(ptr::null(), size as u64, PROT_READ, MAP_SHARED, file.hand, 0);
        if map.to_uint() == -1 { return Err("Failed to map file"); }
        Ok(Mapping{data: map, size: size})
    }
    pub fn open(path: &Path) -> Result<MapFile, &'static str> {
        let file = try!(open_file(path));
        let size = try!(file_size(path));
        let map = try!(map_file(&file, size));
        Ok(MapFile{file: file, map: map})
    }
}<commit_msg>nx::mmap[unix]: Make MapFile public<commit_after>

#[cfg(windows)]
mod mmap {
    use libc::{CreateFileW, CreateFileMappingW, MapViewOfFile, CloseHandle, UnmapViewOfFile,
               GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING, INVALID_HANDLE_VALUE, PAGE_READONLY,
               FILE_MAP_READ, HANDLE, TRUE};
    use libc::consts::os::extra::{FILE_FLAG_RANDOM_ACCESS};
    use libc::types::os::arch::extra::{LPCVOID};
    use std::ptr;
    struct Handle {
        hand: HANDLE,
    }
    impl Drop for Handle {
        fn drop(&mut self) {
            assert_eq!(unsafe { CloseHandle(self.hand) }, TRUE);
        }
    }
    struct MapView {
        data: LPCVOID,
    }
    impl Drop for MapView {
        fn drop(&mut self) {
            assert_eq!(unsafe { UnmapViewOfFile(self.data) }, TRUE);
        }
    }
    pub struct MapFile {
        view: MapView,
        map: Handle,
        file: Handle,
    }
    impl MapFile {
        pub fn data(&self) -> LPCVOID { self.view.data }
    }
    fn create_file(path: &Path) -> Result<Handle, &'static str> {
        let name = path.as_str().unwrap().to_utf16().append([0]);
        let file = unsafe {
            CreateFileW(name.as_ptr(), GENERIC_READ, FILE_SHARE_READ, ptr::mut_null(),
                        OPEN_EXISTING, FILE_FLAG_RANDOM_ACCESS, ptr::mut_null())
        };
        if file.to_uint() == INVALID_HANDLE_VALUE as uint { return Err("Failed to open file"); }
        Ok(Handle{hand: file})
    }
    fn create_mapping(file: &Handle) -> Result<Handle, &'static str> {
        let map = unsafe {
            CreateFileMappingW(file.hand, ptr::mut_null(), PAGE_READONLY, 0, 0, ptr::null())
        };
        if map.is_null() { return Err("Failed to create file mapping"); }
        Ok(Handle{hand: map})
    }
    fn map_view(map: &Handle) -> Result<MapView, &'static str> {
        let view = unsafe { MapViewOfFile(map.hand, FILE_MAP_READ, 0, 0, 0) };
        if view.is_null() { return Err("Failed to map view of file"); }
        Ok(MapView{data: view as *_})
    }
    pub fn open(path: &Path) -> Result<MapFile, &'static str> {
        let file = try!(create_file(path));
        let map = try!(create_mapping(&file));
        let view = try!(map_view(&map));
        Ok(MapFile{file: file, map: map, view: view})
    }
}
#[cfg(unix)]
mod mmap {
    use libc::{close, fstat, mmap, munmap, O_RDONLY, PROT_READ, stat, c_int, c_void};
    use libc::consts::os::posix88::{MAP_SHARED};
    use posix_open = libc::open;
    use std::mem::{uninitialized, transmute};
    use std::ptr;
    use std::io::fs::stat;
    struct Handle {
        hand: c_int,
    }
    impl Drop for Handle {
        fn drop(&mut self) {
            assert_eq!(unsafe { close(self.hand) }, 0);
        }
    }
    struct Mapping {
        data: *mut c_void,
        size: u64,
    }
    impl Drop for Mapping {
        fn drop(&mut self) {
            assert_eq!(unsafe { munmap(self.data as *c_void, self.size) }, 0);
        }
    }
    pub struct MapFile {
        map: Mapping,
        file: Handle,
    }
    impl MapFile {
        pub fn data(&self) -> * c_void { self.map.data as *_ }
    }
    fn open_file(path: &Path) -> Result<Handle, &'static str> {
        let name = path.to_c_str();
        let handle = posix_open(name.unwrap(), O_RDONLY, 0);
        if (handle == -1) { return Err("Failed to open file") }
        Ok(Handle{hand: handle})
    }
    fn file_size(path: &Path) -> Result<u64, &'static str> {
        match path.stat() {
            Ok(stat) => Ok(stat.size),
            Err(_) => Err("Failed to get file size")
        }
    }
    fn map_file(file: &Handle, size: u64) -> Result<Mapping, &'static str> {
        let map = mmap(ptr::null(), size as u64, PROT_READ, MAP_SHARED, file.hand, 0);
        if map.to_uint() == -1 { return Err("Failed to map file"); }
        Ok(Mapping{data: map, size: size})
    }
    pub fn open(path: &Path) -> Result<MapFile, &'static str> {
        let file = try!(open_file(path));
        let size = try!(file_size(path));
        let map = try!(map_file(&file, size));
        Ok(MapFile{file: file, map: map})
    }
}
{'fraction_non_alphanumeric': 0.14769417475728155, 'fraction_numerical': 0.005825242718446602, 'mean_word_length': 2.3459196102314253, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2411742', 'n_tokens_mistral': 2825, 'n_tokens_neox': 2777, 'n_words': 676}