<commit_before>use std::{cmp, ptr};

use self::ApplyResult::*;
use super::Leading::{self, Top, Bot};

const BCNT: u8 = 5;
const BCNT_: usize = BCNT as usize;

#[derive(Copy, Clone)]
struct SPart(u64);

#[derive(Copy, Clone)]
pub struct SPair {
    a: SPart,
    b: SPart,
}

/// A variable length pair.
#[derive(Clone)]
pub struct VPair {
    data: [u64; BCNT_],
    /// Who is leading.
    leading: Leading,
    /// How many bits of the first block are shared.
    prefix: u8,
    /// How many bits of the last block are used.
    used: u8,
    /// which block is last used one
    tail: u8,
}

#[derive(Copy, Clone)]
struct VHead {
    data: u64,
    /// How many bits are shared
    prefix: u8,
    /// How many bits are used
    used: u8,
}

impl VPair {
    pub fn new() -> VPair {
        VPair {
            data: [0; BCNT_],
            prefix: 0,
            used: 0,
            leading: Top,
            tail: 0,
        }
    }

    pub fn len(&self) -> usize {
        self.tail as usize + 1
    }

    pub fn is_complete(&self) -> bool {
        self.tail == 0 && self.used == 0
    }

    pub fn apply(&self, p: &SPair) -> Option<VPair> {
        let (mut lead, mut follow) = match self.leading {
            Top => (p.a, p.b),
            Bot => (p.b, p.a),
        };

        let mut head = self.head();

        match head.apply(&mut follow, &mut lead) {
            Mismatch => None,
            Match => Some(self.with_offset_prefix_and_add_lead(0, head.prefix, lead)),
            LeadSwitch => Some(self.switched_with_new_lead(follow)),
            MatchRemaining => {
                let mut head2 = self.head2();

                match head2.apply(&mut follow, &mut lead) {
                    Mismatch => None,
                    Match => Some(self.with_offset_prefix_and_add_lead(1, head2.prefix, lead)),
                    LeadSwitch => Some(self.switched_with_new_lead(follow)),
                    MatchRemaining => unreachable!("SPairs contain at most 56 bit, which always fit in 64 bit"),
                }
            }
        }
    }

    fn with_offset_prefix_and_add_lead(&self, offset: u8, prefix: u8, lead: SPart) -> VPair {
        let mut p = VPair {
            data: [0; BCNT_],
            prefix: prefix,
            used: self.used,
            leading: self.leading,
            tail: self.tail - offset
        };

        unsafe {
            ptr::copy_nonoverlapping(&self.data[offset as usize], &mut p.data[0], BCNT_ - offset as usize)

        }

        p.apply_lead(lead);

        p
    }

    fn switched_with_new_lead(&self, lead: SPart) -> VPair {
        let mut p = VPair {
            data: [0; BCNT_],
            prefix: 0,
            used: lead.len(),
            leading: self.leading.switched(),
            tail: 0,
        };

        p.data[0] = lead.data();

        p
    }

    fn apply_lead(&mut self, mut lead: SPart) {
        let pushable;

        { // make the borrowchecker happy
            let last = &mut self.data[self.tail as usize];

            // push all we can into the current byte
            pushable = cmp::min(64 - self.used, lead.len());
            *last |= lead.shift_data(pushable) << self.used;
        }

        let new_used = self.used + pushable;

        // if new_used is 64 need to open a new block
        if lead.len() == 0 && new_used < 64 {
            self.used = new_used;
        } else {
            self.push(lead.data());
            self.used = lead.len();
        }

        debug_assert!(self.used != 64);
    }

    fn push(&mut self, block: u64) {
        assert!(self.tail + 1 < BCNT, "lead too long, increase BCNT");
        self.tail += 1;
        self.data[self.tail as usize] = block;
    }

    fn head(&self) -> VHead {
        debug_assert!(self.used != 64);

        VHead {
            data: self.data[0],
            prefix: self.prefix,
            used: if self.tail > 0 { 64 } else { self.used },
        }
    }

    fn head2(&self) -> VHead {
        debug_assert!(self.tail > 0);

        VHead {
            data: self.data[1],
            prefix: 0,
            used: if self.tail > 1 { 64 } else { self. used },
        }
    }
}

enum ApplyResult {
    /// No match
    Mismatch,
    /// Completely matched, bits in lead need to be appended
    Match,
    /// Matched but end of block reached, repeat with second block
    MatchRemaining,
    /// Completely caught up, bits in follow make up a new block
    LeadSwitch,
}

impl SPart {
    fn data(&self) -> u64 {
        self.0 & 0x00FF_FFFF_FFFF_FFFF
    }

    fn len(&self) -> u8 {
        (self.0 >> 56) as u8
    }

    fn shift(&mut self, shift: u8) {
        let new_len = self.len() - shift;
        let new_val = self.data() >> shift;
        self.0 = ((new_len as u64) << 56) | new_val;<commit_msg>add Copy to some types<commit_after>use std::{cmp, ptr};

use self::ApplyResult::*;
use super::Leading::{self, Top, Bot};

const BCNT: u8 = 5;
const BCNT_: usize = BCNT as usize;

#[derive(Copy, Clone)]
struct SPart(u64);

#[derive(Copy, Clone)]
pub struct SPair {
    a: SPart,
    b: SPart,
}

/// A variable length pair.
#[derive(Copy, Clone)]
pub struct VPair {
    data: [u64; BCNT_],
    /// Who is leading.
    leading: Leading,
    /// How many bits of the first block are shared.
    prefix: u8,
    /// How many bits of the last block are used.
    used: u8,
    /// which block is last used one
    tail: u8,
}

#[derive(Copy, Clone)]
struct VHead {
    data: u64,
    /// How many bits are shared
    prefix: u8,
    /// How many bits are used
    used: u8,
}

impl VPair {
    pub fn new() -> VPair {
        VPair {
            data: [0; BCNT_],
            prefix: 0,
            used: 0,
            leading: Top,
            tail: 0,
        }
    }

    pub fn len(&self) -> usize {
        self.tail as usize + 1
    }

    pub fn is_complete(&self) -> bool {
        self.tail == 0 && self.used == 0
    }

    pub fn apply(&self, p: &SPair) -> Option<VPair> {
        let (mut lead, mut follow) = match self.leading {
            Top => (p.a, p.b),
            Bot => (p.b, p.a),
        };

        let mut head = self.head();

        match head.apply(&mut follow, &mut lead) {
            Mismatch => None,
            Match => Some(self.with_offset_prefix_and_add_lead(0, head.prefix, lead)),
            LeadSwitch => Some(self.switched_with_new_lead(follow)),
            MatchRemaining => {
                let mut head2 = self.head2();

                match head2.apply(&mut follow, &mut lead) {
                    Mismatch => None,
                    Match => Some(self.with_offset_prefix_and_add_lead(1, head2.prefix, lead)),
                    LeadSwitch => Some(self.switched_with_new_lead(follow)),
                    MatchRemaining => unreachable!("SPairs contain at most 56 bit, which always fit in 64 bit"),
                }
            }
        }
    }

    fn with_offset_prefix_and_add_lead(&self, offset: u8, prefix: u8, lead: SPart) -> VPair {
        let mut p = VPair {
            data: [0; BCNT_],
            prefix: prefix,
            used: self.used,
            leading: self.leading,
            tail: self.tail - offset
        };

        unsafe {
            ptr::copy_nonoverlapping(&self.data[offset as usize], &mut p.data[0], BCNT_ - offset as usize)

        }

        p.apply_lead(lead);

        p
    }

    fn switched_with_new_lead(&self, lead: SPart) -> VPair {
        let mut p = VPair {
            data: [0; BCNT_],
            prefix: 0,
            used: lead.len(),
            leading: self.leading.switched(),
            tail: 0,
        };

        p.data[0] = lead.data();

        p
    }

    fn apply_lead(&mut self, mut lead: SPart) {
        let pushable;

        { // make the borrowchecker happy
            let last = &mut self.data[self.tail as usize];

            // push all we can into the current byte
            pushable = cmp::min(64 - self.used, lead.len());
            *last |= lead.shift_data(pushable) << self.used;
        }

        let new_used = self.used + pushable;

        // if new_used is 64 need to open a new block
        if lead.len() == 0 && new_used < 64 {
            self.used = new_used;
        } else {
            self.push(lead.data());
            self.used = lead.len();
        }

        debug_assert!(self.used != 64);
    }

    fn push(&mut self, block: u64) {
        assert!(self.tail + 1 < BCNT, "lead too long, increase BCNT");
        self.tail += 1;
        self.data[self.tail as usize] = block;
    }

    fn head(&self) -> VHead {
        debug_assert!(self.used != 64);

        VHead {
            data: self.data[0],
            prefix: self.prefix,
            used: if self.tail > 0 { 64 } else { self.used },
        }
    }

    fn head2(&self) -> VHead {
        debug_assert!(self.tail > 0);

        VHead {
            data: self.data[1],
            prefix: 0,
            used: if self.tail > 1 { 64 } else { self. used },
        }
    }
}

#[derive(Copy, Clone)]
enum ApplyResult {
    /// No match
    Mismatch,
    /// Completely matched, bits in lead need to be appended
    Match,
    /// Matched but end of block reached, repeat with second block
    MatchRemaining,
    /// Completely caught up, bits in follow make up a new block
    LeadSwitch,
}

impl SPart {
    fn data(&self) -> u64 {
        self.0 & 0x00FF_FFFF_FFFF_FFFF
    }

    fn len(&self) -> u8 {
        (self.0 >> 56) as u8
    }

    fn shift(&mut self, shift: u8) {
        let new_len = self.len() - shift;
        let new_val = self.data() >> shift;
        self.0 = ((new_len as u64) << 56) | new_val;
{'fraction_non_alphanumeric': 0.12906904416338377, 'fraction_numerical': 0.016794526228488493, 'mean_word_length': 1.620043454644215, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1912266', 'n_tokens_mistral': 3229, 'n_tokens_neox': 3084, 'n_words': 894}