<commit_before>#cython: embedsignature=True, boundscheck=False, wraparound=False
'''
MMap Arrays: Fast, Disk-Backed Arrays for Python

(c) 2014 Farsight Security Inc.
(c) 2010 <NAME>
'''

cimport cython
from libc.stdint cimport *

import ctypes
import ctypes.util
import os
import platform

import pkg_resources
import six

cdef extern from "mmap_writer.h" nogil:
    cdef void * map_file_ro(int fd, size_t filesize, int want_populate, int want_lock) except NULL
    cdef void * map_file_rw(int fd, size_t filesize, int want_populate, int want_lock) except NULL
    cdef int open_mmap_file_ro(char * filepath) except -1
    cdef int open_mmap_file_rw(char * filename, size_t bytesize, int want_fallocate) except -1
    cdef int close_file(int fd) except -1
    cdef int flush_to_disk(int fd) except -1
    cdef int unmap_file(void * map, int filesize) except -1

_typecodes = {
        'o' : MMapBitArray,
        'b' : MMapInt8Array,
        'B' : MMapUint8Array,
        'h' : MMapInt16Array,
        'H' : MMapUint16Array,
        'i' : MMapInt32Array,
        'I' : MMapUint32Array,
        'l' : MMapInt64Array,
        'L' : MMapUint64Array,
        'f' : MMapFloatArray,
        'd' : MMapDoubleArray,
        }

def array(filename, typecode, size_t size=0, read_only=False, want_fallocate=False, want_populate=False, want_lock=False):
    '''
    Constructs an MMap Array of type "typecode."

    Valid Typecodes:
    'o': boolean
    'b': int8_t
    'B': uint8_t
    'h': int16_t
    'H': uint16_t
    'i': int32_t
    'I': uint32_t
    'l': int64_t
    'L': uint64_t
    'f': float
    'd': double
    '''
    if typecode in _typecodes:
        return _typecodes[typecode](filename, size, read_only, want_fallocate, want_populate, want_lock)
    else:
        raise ValueError('Invalid typecode: \'{}\''.format(typecode))

cdef nmemb(filename, size):
    return <size_t>(os.stat(filename).st_size / size)

cdef class MMapArray:
    """
    Abstract class for MMap Arrays.
    """
    cdef void * _buffer
    cdef size_t _bytesize
    cdef size_t _size
    cdef int _fd
    cdef bytes _filename

    def __init__(self, filename, size_t size=0, read_only=False, want_fallocate=False, want_populate=False, want_lock=False):
        if isinstance(filename, six.string_types):
            self._filename = six.b(filename)
        else:
            self._filename = filename

        self._bytesize = size

        if want_fallocate:
            if platform.system() == 'Linux':
                if pkg_resources.parse_version(platform.release()) < pkg_resources.parse_version('2.6.23'):
                    raise ValueError('fallocate is only available on Linux >= 2.6.23')

                if platform.libc_ver()[0] != 'glibc':
                    raise ValueError('fallocate is only available on glibc')

                libc = ctypes.cdll.LoadLibrary(ctypes.util.find_library('c'))
                gnu_get_libc_version = libc.gnu_get_libc_version
                gnu_get_libc_version.restype = ctypes.c_char_p

                libc_version = gnu_get_libc_version()
                if six.PY3:
                    libc_version = libc_version.decode('ascii')

                if pkg_resources.parse_version(libc_version) < pkg_resources.parse_version('2.10'):
                    raise ValueError('fallocate is only available on glibc >= 2.10')
            else:
                raise ValueError('fallocate is only available on Linux')


        if want_populate and (platform.system() != 'Linux' or platform.system() == 'Linux' and pkg_resources.parse_version(platform.release()) < pkg_resources.parse_version('2.5.46')):
            raise ValueError('MAP_POPULATE is only available on Linux >= 2.5.46')
        if want_lock and (platform.system() != 'Linux' or platform.system() == 'Linux' and pkg_resources.parse_version(platform.release()) < pkg_resources.parse_version('2.5.37')):
            raise ValueError('MAP_LOCKED is only available on Linux >= 2.5.37')

        if read_only:
            self._fd = open_mmap_file_ro(self._filename)

            if os.fstat(self._fd).st_size < self._bytesize:
                raise ValueError('Read-only file too short.  {} < {}'.format(os.fstat(self._fd).st_size, self._bytesize))

            self._buffer = map_file_ro(self._fd, self._bytesize, want_populate, want_lock)
        else:
            self._fd = open_mmap_file_rw(self._filename, self._bytesize, want_fallocate)
            self._buffer = map_file_rw(self._fd, self._bytesize, want_populate, want_lock)
<commit_msg>Clean up want_lock and want_populate checks<commit_after>#cython: embedsignature=True, boundscheck=False, wraparound=False
'''
MMap Arrays: Fast, Disk-Backed Arrays for Python

(c) 2014 Farsight Security Inc.
(c) 2010 <NAME>
'''

cimport cython
from libc.stdint cimport *

import ctypes
import ctypes.util
import os
import platform

import pkg_resources
import six

cdef extern from "mmap_writer.h" nogil:
    cdef void * map_file_ro(int fd, size_t filesize, int want_populate, int want_lock) except NULL
    cdef void * map_file_rw(int fd, size_t filesize, int want_populate, int want_lock) except NULL
    cdef int open_mmap_file_ro(char * filepath) except -1
    cdef int open_mmap_file_rw(char * filename, size_t bytesize, int want_fallocate) except -1
    cdef int close_file(int fd) except -1
    cdef int flush_to_disk(int fd) except -1
    cdef int unmap_file(void * map, int filesize) except -1

_typecodes = {
        'o' : MMapBitArray,
        'b' : MMapInt8Array,
        'B' : MMapUint8Array,
        'h' : MMapInt16Array,
        'H' : MMapUint16Array,
        'i' : MMapInt32Array,
        'I' : MMapUint32Array,
        'l' : MMapInt64Array,
        'L' : MMapUint64Array,
        'f' : MMapFloatArray,
        'd' : MMapDoubleArray,
        }

def array(filename, typecode, size_t size=0, read_only=False, want_fallocate=False, want_populate=False, want_lock=False):
    '''
    Constructs an MMap Array of type "typecode."

    Valid Typecodes:
    'o': boolean
    'b': int8_t
    'B': uint8_t
    'h': int16_t
    'H': uint16_t
    'i': int32_t
    'I': uint32_t
    'l': int64_t
    'L': uint64_t
    'f': float
    'd': double
    '''
    if typecode in _typecodes:
        return _typecodes[typecode](filename, size, read_only, want_fallocate, want_populate, want_lock)
    else:
        raise ValueError('Invalid typecode: \'{}\''.format(typecode))

cdef nmemb(filename, size):
    return <size_t>(os.stat(filename).st_size / size)

cdef class MMapArray:
    """
    Abstract class for MMap Arrays.
    """
    cdef void * _buffer
    cdef size_t _bytesize
    cdef size_t _size
    cdef int _fd
    cdef bytes _filename

    def __init__(self, filename, size_t size=0, read_only=False, want_fallocate=False, want_populate=False, want_lock=False):
        if isinstance(filename, six.string_types):
            self._filename = six.b(filename)
        else:
            self._filename = filename

        self._bytesize = size

        if want_fallocate:
            if platform.system() == 'Linux':
                if pkg_resources.parse_version(platform.release()) < pkg_resources.parse_version('2.6.23'):
                    raise ValueError('fallocate is only available on Linux >= 2.6.23')

                if platform.libc_ver()[0] != 'glibc':
                    raise ValueError('fallocate is only available on glibc')

                libc = ctypes.cdll.LoadLibrary(ctypes.util.find_library('c'))
                gnu_get_libc_version = libc.gnu_get_libc_version
                gnu_get_libc_version.restype = ctypes.c_char_p

                libc_version = gnu_get_libc_version()
                if six.PY3:
                    libc_version = libc_version.decode('ascii')

                if pkg_resources.parse_version(libc_version) < pkg_resources.parse_version('2.10'):
                    raise ValueError('fallocate is only available on glibc >= 2.10')
            else:
                raise ValueError('fallocate is only available on Linux')


        if want_populate:
            if platform.system() != 'Linux':
                raise ValueError('MAP_POPULATE is only available on Linux')

            if pkg_resources.parse_version(platform.release()) < pkg_resources.parse_version('2.5.46'):
                raise ValueError('MAP_POPULATE is only available on Linux >= 2.5.46')

        if want_lock:
            if platform.system() != 'Linux':
                raise ValueError('MAP_LOCKED is only available on Linux')

            if pkg_resources.parse_version(platform.release()) < pkg_resources.parse_version('2.5.37'):
                raise ValueError('MAP_LOCKED is only available on Linux >= 2.5.37')

        if read_only:
            self._fd = open_mmap_file_ro(self._filename)

            if os.fstat(self._fd).st_size < self._bytesize:
                raise ValueError('Read-only file too short.  {} < {}'.format(os.fstat(self._fd).st_size, self._bytesize))

            self._buffer = map_file_ro(self._fd, self._bytesize, want_populate, want_lock)
        else:
            self._fd = open_mmap_file_rw(self._filename, self._bytesize, want_fallocate)
            self._buffer = map_file_rw(self._fd, self._bytesize, want_populate, want_lock)

{'fraction_non_alphanumeric': 0.10373578776394153, 'fraction_numerical': 0.016242555495397944, 'mean_word_length': 2.6899720335597284, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '303996', 'n_tokens_mistral': 3124, 'n_tokens_neox': 2938, 'n_words': 717}