<commit_before>//! Provides various useful utilities.
use std::io;
use std::fs;
use std::fmt;
use std::env;
use std::panic;
use std::os::unix::io::RawFd;
use std::result::Result as StdResult;
use std::io::{Read, Write};
use std::sync::Mutex;

use pbr;
use regex::Regex;
use chrono::Duration;
use serde::{Serialize, Deserialize, de, ser};

use super::{Result, ResultExt, Error, ErrorKind};

pub const SD_LISTEN_FDS_START: RawFd = 3;

/// Helper for serializing/deserializing addresses in string format
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct Addr(pub u64);

/// Helper for formatting durations.
pub struct HumanDuration(pub Duration);

impl Into<u64> for Addr {
    fn into(self) -> u64 {
        self.0
    }
}

impl<'a> fmt::Display for HumanDuration {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        macro_rules! try_write {
            ($num:expr, $str:expr) => {
                if $num == 1 { return write!(f, "1 {}", $str); }
                else if $num > 1 { return write!(f, "{} {}s", $num, $str); }
            }
        }

        try_write!(self.0.num_hours(), "hour");
        try_write!(self.0.num_minutes(), "minute");
        try_write!(self.0.num_seconds(), "second");
        write!(f, "0 seconds")
    }
}

impl Serialize for Addr {
    fn serialize<S>(&self, serializer: S) -> StdResult<S::Ok, S::Error>
        where S: ser::Serializer
    {
        serializer.serialize_str(&format!("0x{:x}", self.0))
    }
}

impl Deserialize for Addr {
    fn deserialize<D>(deserializer: D) -> StdResult<Addr, D::Error>
        where D: de::Deserializer {
        struct AddrVisitor;

        impl de::Visitor for AddrVisitor {
            type Value = u64;

            fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
                formatter.write_str("an address")
            }

            fn visit_str<E: de::Error>(self, value: &str) -> StdResult<u64, E> {
                if &value[..2] == "0x" {
                    u64::from_str_radix(&value[2..], 16)
                        .map_err(|e| E::custom(e.to_string()))
                } else {
                    u64::from_str_radix(&value, 10)
                        .map_err(|e| E::custom(e.to_string()))
                }
            }

            fn visit_u64<E: de::Error>(self, value: u64) -> StdResult<u64, E> {
                Ok(value)
            }
        }

        deserializer.deserialize_str(AddrVisitor).map(Addr)
    }
}

#[derive(Clone, Debug, Default)]
pub struct RegexFilter {
    patterns: Vec<Regex>,
}

impl Deserialize for RegexFilter {
    fn deserialize<D>(deserializer: D) -> StdResult<RegexFilter, D::Error>
        where D: de::Deserializer {
        struct FilterVisitor;

        impl de::Visitor for FilterVisitor {
            type Value = Vec<Regex>;

            fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
                formatter.write_str("a regular expression filter")
            }

            fn visit_seq<V>(self, mut visitor: V)
                -> StdResult<Vec<Regex>, V::Error>
                where V: de::SeqVisitor
            {
                let mut rv = vec![];
                while let Some(item) = visitor.visit::<String>()? {
                    rv.push(Regex::new(&item)
                        .map_err(|err| {
                            <V::Error as de::Error>::custom(format!(
                                "invalid regular expression '{}': {}", &item, err))
                        })?);
                }
                Ok(rv)
            }

            fn visit_str<E: de::Error>(self, value: &str)
                -> StdResult<Vec<Regex>, E>
            {
                Ok(vec![Regex::new(value)
                    .map_err(|err| {
                        E::custom(format!("invalid regular expression '{}': {}", value, err))
                    })?])
            }
        }

        deserializer.deserialize_seq(FilterVisitor).map(|patterns| {
            RegexFilter { patterns: patterns }
        })
    }
}

impl RegexFilter {
    pub fn matches(&self, value: &str) -> bool {<commit_msg>Support empty patterns or single string patterns<commit_after>//! Provides various useful utilities.
use std::io;
use std::fs;
use std::fmt;
use std::env;
use std::panic;
use std::os::unix::io::RawFd;
use std::result::Result as StdResult;
use std::io::{Read, Write};
use std::sync::Mutex;

use pbr;
use regex::Regex;
use chrono::Duration;
use serde::{Serialize, Deserialize, de, ser};

use super::{Result, ResultExt, Error, ErrorKind};

pub const SD_LISTEN_FDS_START: RawFd = 3;

/// Helper for serializing/deserializing addresses in string format
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct Addr(pub u64);

/// Helper for formatting durations.
pub struct HumanDuration(pub Duration);

impl Into<u64> for Addr {
    fn into(self) -> u64 {
        self.0
    }
}

impl<'a> fmt::Display for HumanDuration {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        macro_rules! try_write {
            ($num:expr, $str:expr) => {
                if $num == 1 { return write!(f, "1 {}", $str); }
                else if $num > 1 { return write!(f, "{} {}s", $num, $str); }
            }
        }

        try_write!(self.0.num_hours(), "hour");
        try_write!(self.0.num_minutes(), "minute");
        try_write!(self.0.num_seconds(), "second");
        write!(f, "0 seconds")
    }
}

impl Serialize for Addr {
    fn serialize<S>(&self, serializer: S) -> StdResult<S::Ok, S::Error>
        where S: ser::Serializer
    {
        serializer.serialize_str(&format!("0x{:x}", self.0))
    }
}

impl Deserialize for Addr {
    fn deserialize<D>(deserializer: D) -> StdResult<Addr, D::Error>
        where D: de::Deserializer {
        struct AddrVisitor;

        impl de::Visitor for AddrVisitor {
            type Value = u64;

            fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
                formatter.write_str("an address")
            }

            fn visit_str<E: de::Error>(self, value: &str) -> StdResult<u64, E> {
                if &value[..2] == "0x" {
                    u64::from_str_radix(&value[2..], 16)
                        .map_err(|e| E::custom(e.to_string()))
                } else {
                    u64::from_str_radix(&value, 10)
                        .map_err(|e| E::custom(e.to_string()))
                }
            }

            fn visit_u64<E: de::Error>(self, value: u64) -> StdResult<u64, E> {
                Ok(value)
            }
        }

        deserializer.deserialize_str(AddrVisitor).map(Addr)
    }
}

#[derive(Clone, Debug, Default)]
pub struct RegexFilter {
    patterns: Vec<Regex>,
}

impl Deserialize for RegexFilter {
    fn deserialize<D>(deserializer: D) -> StdResult<RegexFilter, D::Error>
        where D: de::Deserializer {
        struct FilterVisitor;

        fn make_regex<E: de::Error>(value: &str) -> StdResult<Regex, E> {
            Ok(Regex::new(value)
               .map_err(|err| {
                   de::Error::custom(format!(
                       "invalid regular expression '{}': {}", value, err))
               })?)
        }

        impl de::Visitor for FilterVisitor {
            type Value = Vec<Regex>;

            fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
                formatter.write_str("a regular expression filter")
            }

            fn visit_seq<V: de::SeqVisitor>(self, mut visitor: V)
                -> StdResult<Vec<Regex>, V::Error>
            {
                let mut rv = vec![];
                while let Some(item) = visitor.visit::<String>()? {
                    rv.push(make_regex(&item)?);
                }
                Ok(rv)
            }

            fn visit_unit<E: de::Error>(self) -> StdResult<Vec<Regex>, E>
            {
                Ok(vec![])
            }

            fn visit_str<E: de::Error>(self, value: &str)
                -> StdResult<Vec<Regex>, E>
            {
                Ok(vec![make_regex(value)?])
            }
        }

        deserializer.deserialize_seq(FilterVisitor).map(|patterns| {
            RegexFilter { patterns: patterns }
        })
    }
}

impl RegexFilter {
    pub fn matches(&self, value: &str) -> bool {
{'fraction_non_alphanumeric': 0.14903093776333212, 'fraction_numerical': 0.009148910557361261, 'mean_word_length': 1.9388043862752034, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6220739', 'n_tokens_mistral': 2628, 'n_tokens_neox': 2539, 'n_words': 579}