<commit_before>use std::cmp::{max, min};
use std::collections::{HashMap, HashSet};
use std::fmt::Debug;
use utils::calculate_ratio;

#[derive(Debug, Clone, Copy, PartialEq, PartialOrd, Eq, Ord)]
pub struct Match {
    pub first_start: usize,
    pub second_start: usize,
    pub size: usize,
}

impl Match {
    fn new(first_start: usize, second_start: usize, size: usize) -> Match {
        Match {
            first_start: first_start,
            second_start: second_start,
            size: size,
        }
    }
}

#[derive(Debug, Clone, PartialEq)]
pub struct Opcode {
    pub tag: String,
    pub first_start: usize,
    pub first_end: usize,
    pub second_start: usize,
    pub second_end: usize,
}

impl Opcode {
    fn new(
        tag: String,
        first_start: usize,
        first_end: usize,
        second_start: usize,
        second_end: usize,
    ) -> Opcode {
        Opcode {
            tag: tag,
            first_start: first_start,
            first_end: first_end,
            second_start: second_start,
            second_end: second_end,
        }
    }
}

pub trait Sequence: Debug {
    fn len(&self) -> usize;
    fn at_index(&self, index: usize) -> Option<&str>;
}

impl Sequence for str {
    fn len(&self) -> usize {
        self.len()
    }

    fn at_index(&self, index: usize) -> Option<&str> {
        if index > self.len() {
            return None;
        }
        unsafe { Some(self.slice_unchecked(index, index + 1)) }
    }
}

impl<'a> Sequence for Vec<&'a str> {
    fn len(&self) -> usize {
        self.len()
    }

    fn at_index(&self, index: usize) -> Option<&str> {
        if index < self.len() {
            return Some(self[index]);
        }
        None
    }
}

impl Sequence for String {
    fn len(&self) -> usize {
        self.len()
    }

    fn at_index(&self, index: usize) -> Option<&str> {
        if index > self.len() {
            return None;
        }
        unsafe { Some(self.slice_unchecked(index, index + 1)) }
    }
}

impl<'a> Sequence for Vec<String> {
    fn len(&self) -> usize {
        self.len()
    }

    fn at_index(&self, index: usize) -> Option<&str> {
        if index < self.len() {
            return Some(&self[index]);
        }
        None
    }
}

pub struct SequenceMatcher<'a, T: 'a + ?Sized + Sequence> {
    first_sequence: &'a T,
    second_sequence: &'a T,
    matching_blocks: Option<Vec<Match>>,
    opcodes: Option<Vec<Opcode>>,
    is_junk: Option<fn(&str) -> bool>,
    second_sequence_elements: HashMap<&'a str, Vec<usize>>,
    second_sequence_popular: HashSet<&'a str>,
}

impl<'a, T: ?Sized + Sequence> SequenceMatcher<'a, T> {
    pub fn new(first_sequence: &'a T, second_sequence: &'a T) -> SequenceMatcher<'a, T> {
        let mut matcher = SequenceMatcher {
            first_sequence: first_sequence,
            second_sequence: second_sequence,
            matching_blocks: None,
            opcodes: None,
            is_junk: None,
            second_sequence_elements: HashMap::new(),
            second_sequence_popular: HashSet::new(),
        };
        matcher.set_seqs(first_sequence, second_sequence);
        matcher
    }

    pub fn set_is_junk(&mut self, is_junk: Option<fn(&str) -> bool>) {
        self.is_junk = is_junk;
        self.set_second_seq(self.second_sequence);
    }

    pub fn set_seqs(&mut self, first_sequence: &'a T, second_sequence: &'a T) {
        self.set_first_seq(first_sequence);
        self.set_second_seq(second_sequence);
    }

    pub fn set_first_seq(&mut self, sequence: &'a T) {
        self.first_sequence = sequence;
        self.matching_blocks = None;
        self.opcodes = None;
    }

    pub fn set_second_seq(&mut self, sequence: &'a T) {
        self.second_sequence = sequence;
        self.matching_blocks = None;
        self.opcodes = None;
        self.chain_second_seq();
    }

    fn chain_second_seq(&mut self) {
        let second_sequence = self.second_sequence;
        let mut second_sequence_elements = HashMap::new();
        for i in 0..second_sequence.len() {
            let mut counter = second_sequence_elements
                .entry(second_sequence.at_index(i).unwrap())
                .or_insert(Vec::new());
            counter.push(i);
        }
        if self.is_junk.is_some() {
            let mut junk = Vec::new();
            for element in second_sequence_elements.keys() {
                if (self.is_junk.unwrap())(element) {
                    junk.push(element.clone());
                }
            }
            for element in &junk {
                second_sequence_elements.remove(element);
            }
        }
        let mut popular = HashSet::new();
        let len = second_sequence.len();
        if len >= 200 {
            let test_len = (len as f32 / 100.0).floor() as usize + 1;
            for (element, indexes) in second_sequence_elements.iter() {
                if indexes.len() > test_len {
                    popular.insert(element.clone());
                }
            }
            for element in &popular {
                second_sequence_elements.remove(element);
            }
        }
        self.second_sequence_elements = second_sequence_elements;
        self.second_sequence_popular = popular;
    }

    pub fn find_longest_match(
        &self,
        first_start: usize,
        first_end: usize,
        second_start: usize,
        second_end: usize,
    ) -> Match {
        let first_sequence = &self.first_sequence;
        let second_sequence = &self.second_sequence;
        let second_sequence_elements = &self.second_sequence_elements;
        let (mut best_i, mut best_j, mut best_size) = (first_start, second_start, 0);
        let mut j2len: HashMap<usize, usize> = HashMap::new();
        for i in first_start..first_end {
            let mut new_j2len: HashMap<usize, usize> = HashMap::new();
            match second_sequence_elements.get(first_sequence.at_index(i).unwrap()) {
                Some(indexes) => {
                    for j in indexes {
                        let j = j.clone();
                        if j < second_start {
                            continue;<commit_msg>Remove second_sequence_popular from the SequenceMatcher struct.<commit_after>use std::cmp::{max, min};
use std::collections::{HashMap, HashSet};
use std::fmt::Debug;
use utils::calculate_ratio;

#[derive(Debug, Clone, Copy, PartialEq, PartialOrd, Eq, Ord)]
pub struct Match {
    pub first_start: usize,
    pub second_start: usize,
    pub size: usize,
}

impl Match {
    fn new(first_start: usize, second_start: usize, size: usize) -> Match {
        Match {
            first_start: first_start,
            second_start: second_start,
            size: size,
        }
    }
}

#[derive(Debug, Clone, PartialEq)]
pub struct Opcode {
    pub tag: String,
    pub first_start: usize,
    pub first_end: usize,
    pub second_start: usize,
    pub second_end: usize,
}

impl Opcode {
    fn new(
        tag: String,
        first_start: usize,
        first_end: usize,
        second_start: usize,
        second_end: usize,
    ) -> Opcode {
        Opcode {
            tag: tag,
            first_start: first_start,
            first_end: first_end,
            second_start: second_start,
            second_end: second_end,
        }
    }
}

pub trait Sequence: Debug {
    fn len(&self) -> usize;
    fn at_index(&self, index: usize) -> Option<&str>;
}

impl Sequence for str {
    fn len(&self) -> usize {
        self.len()
    }

    fn at_index(&self, index: usize) -> Option<&str> {
        if index > self.len() {
            return None;
        }
        unsafe { Some(self.slice_unchecked(index, index + 1)) }
    }
}

impl<'a> Sequence for Vec<&'a str> {
    fn len(&self) -> usize {
        self.len()
    }

    fn at_index(&self, index: usize) -> Option<&str> {
        if index < self.len() {
            return Some(self[index]);
        }
        None
    }
}

impl Sequence for String {
    fn len(&self) -> usize {
        self.len()
    }

    fn at_index(&self, index: usize) -> Option<&str> {
        if index > self.len() {
            return None;
        }
        unsafe { Some(self.slice_unchecked(index, index + 1)) }
    }
}

impl<'a> Sequence for Vec<String> {
    fn len(&self) -> usize {
        self.len()
    }

    fn at_index(&self, index: usize) -> Option<&str> {
        if index < self.len() {
            return Some(&self[index]);
        }
        None
    }
}

pub struct SequenceMatcher<'a, T: 'a + ?Sized + Sequence> {
    first_sequence: &'a T,
    second_sequence: &'a T,
    matching_blocks: Option<Vec<Match>>,
    opcodes: Option<Vec<Opcode>>,
    is_junk: Option<fn(&str) -> bool>,
    second_sequence_elements: HashMap<&'a str, Vec<usize>>,
}

impl<'a, T: ?Sized + Sequence> SequenceMatcher<'a, T> {
    pub fn new(first_sequence: &'a T, second_sequence: &'a T) -> SequenceMatcher<'a, T> {
        let mut matcher = SequenceMatcher {
            first_sequence: first_sequence,
            second_sequence: second_sequence,
            matching_blocks: None,
            opcodes: None,
            is_junk: None,
            second_sequence_elements: HashMap::new(),
        };
        matcher.set_seqs(first_sequence, second_sequence);
        matcher
    }

    pub fn set_is_junk(&mut self, is_junk: Option<fn(&str) -> bool>) {
        self.is_junk = is_junk;
        self.set_second_seq(self.second_sequence);
    }

    pub fn set_seqs(&mut self, first_sequence: &'a T, second_sequence: &'a T) {
        self.set_first_seq(first_sequence);
        self.set_second_seq(second_sequence);
    }

    pub fn set_first_seq(&mut self, sequence: &'a T) {
        self.first_sequence = sequence;
        self.matching_blocks = None;
        self.opcodes = None;
    }

    pub fn set_second_seq(&mut self, sequence: &'a T) {
        self.second_sequence = sequence;
        self.matching_blocks = None;
        self.opcodes = None;
        self.chain_second_seq();
    }

    fn chain_second_seq(&mut self) {
        let second_sequence = self.second_sequence;
        let mut second_sequence_elements = HashMap::new();
        for i in 0..second_sequence.len() {
            let mut counter = second_sequence_elements
                .entry(second_sequence.at_index(i).unwrap())
                .or_insert(Vec::new());
            counter.push(i);
        }
        if self.is_junk.is_some() {
            let mut junk = Vec::new();
            for element in second_sequence_elements.keys() {
                if (self.is_junk.unwrap())(element) {
                    junk.push(element.clone());
                }
            }
            for element in &junk {
                second_sequence_elements.remove(element);
            }
        }
        let mut popular = HashSet::new();
        let len = second_sequence.len();
        if len >= 200 {
            let test_len = (len as f32 / 100.0).floor() as usize + 1;
            for (element, indexes) in second_sequence_elements.iter() {
                if indexes.len() > test_len {
                    popular.insert(element.clone());
                }
            }
            for element in &popular {
                second_sequence_elements.remove(element);
            }
        }
        self.second_sequence_elements = second_sequence_elements;
    }

    pub fn find_longest_match(
        &self,
        first_start: usize,
        first_end: usize,
        second_start: usize,
        second_end: usize,
    ) -> Match {
        let first_sequence = &self.first_sequence;
        let second_sequence = &self.second_sequence;
        let second_sequence_elements = &self.second_sequence_elements;
        let (mut best_i, mut best_j, mut best_size) = (first_start, second_start, 0);
        let mut j2len: HashMap<usize, usize> = HashMap::new();
        for i in first_start..first_end {
            let mut new_j2len: HashMap<usize, usize> = HashMap::new();
            match second_sequence_elements.get(first_sequence.at_index(i).unwrap()) {
                Some(indexes) => {
                    for j in indexes {
                        let j = j.clone();
                        if j < second_start {
                            continue;
{'fraction_non_alphanumeric': 0.11708033964728935, 'fraction_numerical': 0.002612671456564337, 'mean_word_length': 1.9701745877788555, 'pattern_counts': {'":': 0, '<': 56, '<?xml version=': 0, '>': 92, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '775141', 'n_tokens_mistral': 3807, 'n_tokens_neox': 3794, 'n_words': 875}