<commit_before>use fnv::FnvHasher;
use std::collections;
use std::hash::BuildHasherDefault;
use std::hash::Hash;
use std::hash::Hasher;
use std::ops::Range;
use std::ptr;
use std::slice;

pub type HashMap<K, V> = collections::HashMap<K, V, BuildHasherDefault<FnvHasher>>;
pub type HashSet<K> = collections::HashSet<K, BuildHasherDefault<FnvHasher>>;

pub fn new_map<K, V>() -> HashMap<K, V>
    where K: Eq + Hash
{
    HashMap::<K, V>::with_hasher(Default::default())
}

pub fn new_set<K>() -> HashSet<K>
    where K: Eq + Hash
{
    HashSet::<K>::with_hasher(Default::default())
}

pub fn ptr_eq<T>(x: &T, y: &T) -> bool {
    x as *const _ == y as *const _
}

// avoids dropping overhead

pub fn fast_clear<T: Copy>(vec: &mut Vec<T>) {
    unsafe {
        vec.set_len(0);
    }
}

pub fn fast_extend<T: Copy>(vec: &mut Vec<T>, other: &[T]) {
    vec.reserve(other.len());
    unsafe {
        let len = vec.len();
        ptr::copy_nonoverlapping(other.get_unchecked(0),
                                 vec.get_unchecked_mut(len),
                                 other.len());
        vec.set_len(len + other.len());
    }
}

pub fn copy_portion(vec: &mut Vec<u8>, from: Range<usize>) {
    let Range { start: copy_start, end: copy_end } = from.clone();
    &vec[from]; // for the bounds check
    unsafe {
        let copy_len = copy_end - copy_start;
        vec.reserve(copy_len);

        let old_len = vec.len();
        let copy_from = vec.as_ptr().offset(copy_start as isize);
        let copy_to = vec.as_mut_ptr().offset(old_len as isize);
        ptr::copy_nonoverlapping(copy_from, copy_to, copy_len);
        vec.set_len(old_len + copy_len);
    }<commit_msg>most memcpies we do are very short<commit_after>use fnv::FnvHasher;
use std::collections;
use std::hash::BuildHasherDefault;
use std::hash::Hash;
use std::hash::Hasher;
use std::ops::Range;
use std::ptr;
use std::slice;

pub type HashMap<K, V> = collections::HashMap<K, V, BuildHasherDefault<FnvHasher>>;
pub type HashSet<K> = collections::HashSet<K, BuildHasherDefault<FnvHasher>>;

pub fn new_map<K, V>() -> HashMap<K, V>
    where K: Eq + Hash
{
    HashMap::<K, V>::with_hasher(Default::default())
}

pub fn new_set<K>() -> HashSet<K>
    where K: Eq + Hash
{
    HashSet::<K>::with_hasher(Default::default())
}

pub fn ptr_eq<T>(x: &T, y: &T) -> bool {
    x as *const _ == y as *const _
}

// avoids dropping overhead

pub fn fast_clear<T: Copy>(vec: &mut Vec<T>) {
    unsafe {
        vec.set_len(0);
    }
}

unsafe fn short_copy<T>(src: *const T, dst: *mut T, count: usize) {
    if count == 2 {
        ptr::write(dst as *mut [T; 2], ptr::read(src as *const [T; 2]));
    } else if count == 1 {
        ptr::write(dst, ptr::read(src));
    } else {
        ptr::copy_nonoverlapping(src, dst, count);
    }
}

pub fn fast_extend<T: Copy>(vec: &mut Vec<T>, other: &[T]) {
    vec.reserve(other.len());
    unsafe {
        let len = vec.len();
        short_copy(other.get_unchecked(0),
                                 vec.get_unchecked_mut(len),
                                 other.len());
        vec.set_len(len + other.len());
    }
}

pub fn copy_portion(vec: &mut Vec<u8>, from: Range<usize>) {
    let Range { start: copy_start, end: copy_end } = from.clone();
    &vec[from]; // for the bounds check
    unsafe {
        let copy_len = copy_end - copy_start;
        vec.reserve(copy_len);

        let old_len = vec.len();
        let copy_from = vec.as_ptr().offset(copy_start as isize);
        let copy_to = vec.as_mut_ptr().offset(old_len as isize);
        short_copy(copy_from, copy_to, copy_len);
        vec.set_len(old_len + copy_len);
    }
{'fraction_non_alphanumeric': 0.16223622910386407, 'fraction_numerical': 0.0027404768429706767, 'mean_word_length': 2.945945945945946, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1294266', 'n_tokens_mistral': 1330, 'n_tokens_neox': 1326, 'n_words': 307}