<commit_before>// TODO: deny missing docs
#![allow(missing_docs)]

use dns::{Response, Result as LookupResult, Error as LookupError, Source};

use itertools::Itertools;
use std::collections::HashMap;
use trust_dns::rr::{RData, Record};

#[derive(Debug)]
pub struct RecordCount<'a> {
    record: &'a Record,
    sources: Vec<Source>,
}

impl<'a> RecordCount<'a> {
    pub fn count(&self) -> usize {
        self.sources.len()
    }

    pub fn record(&self) -> &Record {
        self.record
    }

    pub fn sources(&self) -> &Vec<Source> {
        &self.sources
    }
}

#[derive(Debug)]
pub struct Summary<'a> {
    pub num_of_samples: usize,
    pub num_of_ok_samples: usize,
    pub num_of_err_samples: usize,
    pub min_num_of_records: usize,
    pub max_num_of_records: usize,
    pub record_counts: HashMap<String, RecordCount<'a>>,
    pub failures: Vec<&'a LookupError>,
    pub alerts: Vec<Alert>
}

impl<'a> Summary<'a> {
    pub fn from(responses: &'a [LookupResult<Response>]) -> Self {
        let num_of_samples = responses.len();

        let (responses, failures): (Vec<_>, Vec<_>) =
            responses.into_iter().partition(|x| x.is_ok());
        let successes: Vec<&Response> =
            responses.into_iter().map(|x| x.as_ref().unwrap()).collect();
        let failures: Vec<&LookupError> = failures
            .into_iter()
            .map(|x| x.as_ref().unwrap_err())
            .collect();

        let num_of_ok_samples = successes.len();
        let num_of_err_samples = failures.len();

        let (min_num_of_records, max_num_of_records) = successes
            .iter()
            .map(|x| x.answers.len())
            .fold((::std::usize::MAX, 0), |(mut min, mut max), x| {
                if min > x {
                    min = x
                };
                if max < x {
                    max = x
                };
                (min, max)
            });

        let mut record_counts = HashMap::new();
        for response in &successes {
            for record in &response.answers {
                let key = format!("{:?}-{:?}", record.rr_type(), record.rdata());
                let value = record_counts.entry(key).or_insert( RecordCount{ record, sources: Vec::new() } );
                value.sources.push(response.server.source);
            }
        }

        let alerts = alerts(successes.as_slice());

        Summary {
            num_of_samples,
            num_of_ok_samples,
            num_of_err_samples,
            min_num_of_records,
            max_num_of_records,
            record_counts,
            failures,
            alerts
        }
    }
}

#[derive(Debug)]
pub enum Alert {
    SoaSnDiverge(HashMap<u32, u32>),
}

// TODO: &[&T] is totally weird -- cf. https://users.rust-lang.org/t/solved-function-taking-slice-of-objects-as-well-as-slice-of-references-to-objects/13553/9
fn alerts(responses: &[&Response]) -> Vec<Alert> {
    let mut alerts = Vec::new();
    if let Some(serials) = check_soa_serial_numbers(responses) {
        alerts.push(serials)
    }

    alerts
}

// TODO: &[&T] is totally weird
fn check_soa_serial_numbers(responses: &[&Response]) -> Option<Alert> {
    let mut serial_counts = HashMap::new();
    responses
        .iter()
        .map(|r| r.answers
            .iter()
            .map(|a| match *a.rdata() {
                RData::SOA(ref soa) => Some(soa.serial()),
                _ => None
            })
        )
        .flatten()
        .filter_map(|x| x)
        .for_each(|key| {
            let value = serial_counts.entry(key).or_insert(0);
            *value += 1;
        });

    if serial_counts.len() > 1 {<commit_msg>Fix regression due to pending stabilization of Iterator::flatten<commit_after>// TODO: deny missing docs
#![allow(missing_docs)]

use dns::{Response, Result as LookupResult, Error as LookupError, Source};

use itertools::Itertools;
use std::collections::HashMap;
use trust_dns::rr::{RData, Record};

#[derive(Debug)]
pub struct RecordCount<'a> {
    record: &'a Record,
    sources: Vec<Source>,
}

impl<'a> RecordCount<'a> {
    pub fn count(&self) -> usize {
        self.sources.len()
    }

    pub fn record(&self) -> &Record {
        self.record
    }

    pub fn sources(&self) -> &Vec<Source> {
        &self.sources
    }
}

#[derive(Debug)]
pub struct Summary<'a> {
    pub num_of_samples: usize,
    pub num_of_ok_samples: usize,
    pub num_of_err_samples: usize,
    pub min_num_of_records: usize,
    pub max_num_of_records: usize,
    pub record_counts: HashMap<String, RecordCount<'a>>,
    pub failures: Vec<&'a LookupError>,
    pub alerts: Vec<Alert>
}

impl<'a> Summary<'a> {
    pub fn from(responses: &'a [LookupResult<Response>]) -> Self {
        let num_of_samples = responses.len();

        let (responses, failures): (Vec<_>, Vec<_>) =
            responses.into_iter().partition(|x| x.is_ok());
        let successes: Vec<&Response> =
            responses.into_iter().map(|x| x.as_ref().unwrap()).collect();
        let failures: Vec<&LookupError> = failures
            .into_iter()
            .map(|x| x.as_ref().unwrap_err())
            .collect();

        let num_of_ok_samples = successes.len();
        let num_of_err_samples = failures.len();

        let (min_num_of_records, max_num_of_records) = successes
            .iter()
            .map(|x| x.answers.len())
            .fold((::std::usize::MAX, 0), |(mut min, mut max), x| {
                if min > x {
                    min = x
                };
                if max < x {
                    max = x
                };
                (min, max)
            });

        let mut record_counts = HashMap::new();
        for response in &successes {
            for record in &response.answers {
                let key = format!("{:?}-{:?}", record.rr_type(), record.rdata());
                let value = record_counts.entry(key).or_insert( RecordCount{ record, sources: Vec::new() } );
                value.sources.push(response.server.source);
            }
        }

        let alerts = alerts(successes.as_slice());

        Summary {
            num_of_samples,
            num_of_ok_samples,
            num_of_err_samples,
            min_num_of_records,
            max_num_of_records,
            record_counts,
            failures,
            alerts
        }
    }
}

#[derive(Debug)]
pub enum Alert {
    SoaSnDiverge(HashMap<u32, u32>),
}

// TODO: &[&T] is totally weird -- cf. https://users.rust-lang.org/t/solved-function-taking-slice-of-objects-as-well-as-slice-of-references-to-objects/13553/9
fn alerts(responses: &[&Response]) -> Vec<Alert> {
    let mut alerts = Vec::new();
    if let Some(serials) = check_soa_serial_numbers(responses) {
        alerts.push(serials)
    }

    alerts
}

// TODO: &[&T] is totally weird
fn check_soa_serial_numbers(responses: &[&Response]) -> Option<Alert> {
    let mut serial_counts = HashMap::new();
    Itertools::flatten(
        responses
        .iter()
        .map(|r| r.answers
            .iter()
            .map(|a| match *a.rdata() {
                RData::SOA(ref soa) => Some(soa.serial()),
                _ => None
            })
        )
    )
    .filter_map(|x| x)
    .for_each(|key| {
        let value = serial_counts.entry(key).or_insert(0);
        *value += 1;
    });

    if serial_counts.len() > 1 {
{'fraction_non_alphanumeric': 0.1456746463547334, 'fraction_numerical': 0.003808487486398259, 'mean_word_length': 2.0561097256857854, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 63, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2304344', 'n_tokens_mistral': 2435, 'n_tokens_neox': 2402, 'n_words': 491}