<commit_before>use std::mem;
use std::ops;
use std::ptr;
use std::slice;
use std::iter;

use nodrop::NoDrop;

use num_traits::{Unsigned, Zero, One};

#[derive(Debug)]
pub struct OverflowError<T1, T2>(pub T1, pub T2);

#[derive(Debug)]
pub enum InsertError<T1, T2> {
    Overflow(T1, T2),
    OutOfBounds(T1, T2)
}

pub unsafe trait Array {
    type Item;
    type Index: Index;
    fn as_ptr(&self) -> *const Self::Item;
    fn as_mut_ptr(&mut self) -> *mut Self::Item;
    fn capacity() -> usize;
}

pub trait Index : Unsigned + ops::AddAssign + ops::SubAssign + Copy {
    fn as_usize(self) -> usize;
    fn from_usize(value: usize) -> Self;
}

macro_rules! impl_index {
    ($index_type:ty) => (
        impl Index for $index_type {
            fn as_usize(self) -> usize {
                self as usize
            }

            fn from_usize(value: usize) -> Self {
                value as $index_type
            }
        }
    )
}

impl_index!(u8);
impl_index!(u16);
impl_index!(u32);
impl_index!(usize);

macro_rules! impl_array {
    ($index_type:ty => $($len:expr),+ $(,)*) => (
        $(
            unsafe impl<T> Array for [T; $len] {
                type Item = T;
                type Index = $index_type;
                #[inline(always)]
                fn as_ptr(&self) -> *const T {
                    self as *const _ as *const _
                }

                #[inline(always)]
                fn as_mut_ptr(&mut self) -> *mut T {
                    self as *mut _ as *mut _
                }

                #[inline(always)]
                fn capacity() -> usize {
                    $len
                }
            }
        )*
    )
}

impl_array!(u8 => 1, 2, 7, 31, 256);

pub struct LockstepArray<A1: Array, A2: Array> {
    len: A1::Index,
    array1: NoDrop<A1>,
    array2: NoDrop<A2>
}

impl<A1, A2> LockstepArray<A1, A2> where A1: Array, A2: Array {
    pub fn new() -> LockstepArray<A1, A2> {
        // TODO: Eventually these will be encoded into the type system
        assert_eq!(A1::capacity(), A2::capacity());
        LockstepArray {
            len: Zero::zero(),
            array1: NoDrop::new(unsafe {mem::uninitialized()}),
            array2: NoDrop::new(unsafe {mem::uninitialized()})
        }
    }

    pub fn from_arrays<B1, B2>(array1: B1, array2: B2) -> LockstepArray<A1, A2>
        where B1: Array<Item=A1::Item>, B2: Array<Item=A2::Item> {
        let mut lockstep = LockstepArray::<A1, A2>::new();
        // TODO: Eventually these will be encoded into the type system
        assert!(B1::capacity() <= A1::capacity());
        assert!(B2::capacity() <= A2::capacity());
        assert_eq!(A1::capacity(), A2::capacity());
        let len = A1::Index::from_usize(A1::capacity());
        unsafe {
            ptr::copy_nonoverlapping(array1.as_ptr(),
                                     lockstep.array1.as_mut_ptr(),
                                     A1::capacity());
            mem::forget(array1);
            ptr::copy_nonoverlapping(array2.as_ptr(),
                                     lockstep.array2.as_mut_ptr(),
                                     A1::capacity());
            mem::forget(array2);
        }
        lockstep.len = len;
        lockstep
    }

    pub fn push(&mut self, item1: A1::Item, item2: A2::Item)<commit_msg>Fix a panic on inserting third item<commit_after>use std::mem;
use std::ops;
use std::ptr;
use std::slice;
use std::iter;

use nodrop::NoDrop;

use num_traits::{Unsigned, Zero, One};

#[derive(Debug)]
pub struct OverflowError<T1, T2>(pub T1, pub T2);

#[derive(Debug)]
pub enum InsertError<T1, T2> {
    Overflow(T1, T2),
    OutOfBounds(T1, T2)
}

pub unsafe trait Array {
    type Item;
    type Index: Index;
    fn as_ptr(&self) -> *const Self::Item;
    fn as_mut_ptr(&mut self) -> *mut Self::Item;
    fn capacity() -> usize;
}

pub trait Index : Unsigned + ops::AddAssign + ops::SubAssign + Copy {
    fn as_usize(self) -> usize;
    fn from_usize(value: usize) -> Self;
}

macro_rules! impl_index {
    ($index_type:ty) => (
        impl Index for $index_type {
            fn as_usize(self) -> usize {
                self as usize
            }

            fn from_usize(value: usize) -> Self {
                value as $index_type
            }
        }
    )
}

impl_index!(u8);
impl_index!(u16);
impl_index!(u32);
impl_index!(usize);

macro_rules! impl_array {
    ($index_type:ty => $($len:expr),+ $(,)*) => (
        $(
            unsafe impl<T> Array for [T; $len] {
                type Item = T;
                type Index = $index_type;
                #[inline(always)]
                fn as_ptr(&self) -> *const T {
                    self as *const _ as *const _
                }

                #[inline(always)]
                fn as_mut_ptr(&mut self) -> *mut T {
                    self as *mut _ as *mut _
                }

                #[inline(always)]
                fn capacity() -> usize {
                    $len
                }
            }
        )*
    )
}

impl_array!(u8 => 1, 2, 7, 31, 256);

pub struct LockstepArray<A1: Array, A2: Array> {
    len: A1::Index,
    array1: NoDrop<A1>,
    array2: NoDrop<A2>
}

impl<A1, A2> LockstepArray<A1, A2> where A1: Array, A2: Array {
    pub fn new() -> LockstepArray<A1, A2> {
        // TODO: Eventually these will be encoded into the type system
        assert_eq!(A1::capacity(), A2::capacity());
        LockstepArray {
            len: Zero::zero(),
            array1: NoDrop::new(unsafe {mem::uninitialized()}),
            array2: NoDrop::new(unsafe {mem::uninitialized()})
        }
    }

    pub fn from_arrays<B1, B2>(array1: B1, array2: B2) -> LockstepArray<A1, A2>
        where B1: Array<Item=A1::Item>, B2: Array<Item=A2::Item> {
        let mut lockstep = LockstepArray::<A1, A2>::new();
        // TODO: Eventually these will be encoded into the type system
        assert!(B1::capacity() <= A1::capacity());
        assert!(B2::capacity() <= A2::capacity());
        assert_eq!(B1::capacity(), B2::capacity());
        let len = A1::Index::from_usize(B1::capacity());
        unsafe {
            ptr::copy_nonoverlapping(array1.as_ptr(),
                                     lockstep.array1.as_mut_ptr(),
                                     B1::capacity());
            mem::forget(array1);
            ptr::copy_nonoverlapping(array2.as_ptr(),
                                     lockstep.array2.as_mut_ptr(),
                                     B2::capacity());
            mem::forget(array2);
        }
        lockstep.len = len;
        lockstep
    }

    pub fn push(&mut self, item1: A1::Item, item2: A2::Item)
{'fraction_non_alphanumeric': 0.14564564564564564, 'fraction_numerical': 0.02312312312312312, 'mean_word_length': 1.845365228534814, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4119421', 'n_tokens_mistral': 2214, 'n_tokens_neox': 2161, 'n_words': 495}