<commit_before>extern crate chrono;
extern crate lopdf;
extern crate png;
extern crate predicates;

pub mod file {

    use chrono::{DateTime, FixedOffset, Utc};

    use predicates::boolean::AndPredicate;
    use predicates::prelude::*;
    use predicates::reflection::{Case, Child, PredicateReflection, Product};
    use predicates::str::{ContainsPredicate, StartsWithPredicate};

    use std::fmt;

    /// Checks that the variable of type [u8] can be parsed as a PDF file.
    #[derive(Debug)]
    pub struct PdfPredicate {}

    impl PdfPredicate {
        pub fn with_page_count(self: Self, num_pages: usize) -> DetailPredicate<Self> {
            DetailPredicate::<Self> {
                p: self,
                d: Detail::PageCount(num_pages),
            }
        }

        pub fn with_creation_date(self: Self, when: DateTime<Utc>) -> DetailPredicate<Self> {
            DetailPredicate::<Self> {
                p: self,
                d: Detail::CreationDate(when),
            }
        }
    }

    impl Predicate<[u8]> for PdfPredicate {
        fn eval(&self, data: &[u8]) -> bool {
            lopdf::Document::load_mem(data).is_ok()
        }

        fn find_case<'a>(&'a self, _expected: bool, data: &[u8]) -> Option<Case<'a>> {
            match lopdf::Document::load_mem(data) {
                Ok(_) => None,
                Err(e) => Some(Case::new(Some(self), false).add_product(Product::new("Error", e))),
            }
        }
    }

    impl PredicateReflection for PdfPredicate {}

    impl fmt::Display for PdfPredicate {
        fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
            write!(f, "is a PDF")
        }
    }

    /// Extends a PdfPredicate by a check for page count or creation date.
    #[derive(Debug)]
    pub struct DetailPredicate<PdfPredicate> {
        p: PdfPredicate,
        d: Detail,
    }

    #[derive(Debug)]
    enum Detail {
        PageCount(usize),
        CreationDate(DateTime<Utc>),
    }

    trait Details {
        fn get_num_pages(&self) -> usize;
        fn get_creation_date(&self) -> Option<DateTime<Utc>>;
    }

    impl DetailPredicate<PdfPredicate> {
        fn eval_doc(&self, doc: &lopdf::Document) -> bool {
            match self.d {
                Detail::PageCount(n) => n == doc.get_num_pages(),
                Detail::CreationDate(d) => doc.get_creation_date().map_or(false, |date| date == d),
            }
        }

        fn find_case_for_doc<'a>(
            &'a self,
            expected: bool,
            doc: &lopdf::Document,
        ) -> Option<Case<'a>> {
            if self.eval_doc(doc) == expected {
                let product = self.product_for_doc(doc);
                Some(Case::new(Some(self), false).add_product(product))
            } else {
                None
            }
        }

        fn product_for_doc(&self, doc: &lopdf::Document) -> Product {
            match self.d {
                Detail::PageCount(_) => Product::new(
                    "actual page count",
                    format!("{} page(s)", doc.get_num_pages()),
                ),
                Detail::CreationDate(_) => Product::new(
                    "actual creation date",
                    format!("{:?}", doc.get_creation_date()),
                ),
            }
        }
    }

    impl Details for lopdf::Document {
        fn get_creation_date(self: &Self) -> Option<DateTime<Utc>> {
            fn get_from_trailer<'a>(
                doc: &'a lopdf::Document,
                key: &[u8],
            ) -> lopdf::Result<&'a lopdf::Object> {
                let id = doc.trailer.get(b"Info")?.as_reference()?;
                doc.get_object(id)?.as_dict()?.get(key)
            }

            if let Ok(obj) = get_from_trailer(self, b"CreationDate") {
                // Now this should actually be as simple as returning obj.as_datetime().
                // However there are bugs that need to be worked around here:
                //
                // First of all cairo inadvertently truncates the timezone offset,
                // see https://gitlab.freedesktop.org/cairo/cairo/issues/392
                //
                // On top of that the lopdf::Object::as_datetime() method has issues
                // and can not be used, see https://github.com/J-F-Liu/lopdf/issues/88
                //
                // So here's our implentation instead.

                fn as_datetime(str: &str) -> Option<DateTime<FixedOffset>> {
                    if str.ends_with("0000") {
                        DateTime::parse_from_str(str, "%Y%m%d%H%M%S%z").ok()
                    } else {
                        let str = String::from(str) + "00";
                        as_datetime(&str)
                    }
                }

                if let lopdf::Object::String(ref bytes, _) = obj {
                    if let Ok(str) = String::from_utf8(
                        bytes
                            .iter()
                            .filter(|b| ![b'D', b':', b'\''].contains(b))
                            .cloned()
                            .collect(),
                    ) {
                        return as_datetime(&str).map(|date| date.with_timezone(&Utc));
                    }
                }
            }
            None
        }

        fn get_num_pages(self: &Self) -> usize {
            self.get_pages().len()
        }
    }

    impl Predicate<[u8]> for DetailPredicate<PdfPredicate> {
        fn eval(&self, data: &[u8]) -> bool {
            match lopdf::Document::load_mem(data) {
                Ok(doc) => self.eval_doc(&doc),
                _ => false,
            }
        }

        fn find_case<'a>(&'a self, expected: bool, data: &[u8]) -> Option<Case<'a>> {
            match lopdf::Document::load_mem(data) {
                Ok(doc) => self.find_case_for_doc(expected, &doc),
                Err(e) => Some(Case::new(Some(self), false).add_product(Product::new("Error", e))),
            }
        }
    }

    impl PredicateReflection for DetailPredicate<PdfPredicate> {
        fn children<'a>(&'a self) -> Box<dyn Iterator<Item = Child<'a>> + 'a> {
            let params = vec![Child::new("predicate", &self.p)];
            Box::new(params.into_iter())
        }
    }<commit_msg>Remove obsolete workarounds for datetime parsing<commit_after>extern crate chrono;
extern crate lopdf;
extern crate png;
extern crate predicates;

pub mod file {

    use chrono::{DateTime, Utc};

    use predicates::boolean::AndPredicate;
    use predicates::prelude::*;
    use predicates::reflection::{Case, Child, PredicateReflection, Product};
    use predicates::str::{ContainsPredicate, StartsWithPredicate};

    use std::fmt;

    /// Checks that the variable of type [u8] can be parsed as a PDF file.
    #[derive(Debug)]
    pub struct PdfPredicate {}

    impl PdfPredicate {
        pub fn with_page_count(self: Self, num_pages: usize) -> DetailPredicate<Self> {
            DetailPredicate::<Self> {
                p: self,
                d: Detail::PageCount(num_pages),
            }
        }

        pub fn with_creation_date(self: Self, when: DateTime<Utc>) -> DetailPredicate<Self> {
            DetailPredicate::<Self> {
                p: self,
                d: Detail::CreationDate(when),
            }
        }
    }

    impl Predicate<[u8]> for PdfPredicate {
        fn eval(&self, data: &[u8]) -> bool {
            lopdf::Document::load_mem(data).is_ok()
        }

        fn find_case<'a>(&'a self, _expected: bool, data: &[u8]) -> Option<Case<'a>> {
            match lopdf::Document::load_mem(data) {
                Ok(_) => None,
                Err(e) => Some(Case::new(Some(self), false).add_product(Product::new("Error", e))),
            }
        }
    }

    impl PredicateReflection for PdfPredicate {}

    impl fmt::Display for PdfPredicate {
        fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
            write!(f, "is a PDF")
        }
    }

    /// Extends a PdfPredicate by a check for page count or creation date.
    #[derive(Debug)]
    pub struct DetailPredicate<PdfPredicate> {
        p: PdfPredicate,
        d: Detail,
    }

    #[derive(Debug)]
    enum Detail {
        PageCount(usize),
        CreationDate(DateTime<Utc>),
    }

    trait Details {
        fn get_num_pages(&self) -> usize;
        fn get_creation_date(&self) -> Option<DateTime<Utc>>;
    }

    impl DetailPredicate<PdfPredicate> {
        fn eval_doc(&self, doc: &lopdf::Document) -> bool {
            match self.d {
                Detail::PageCount(n) => n == doc.get_num_pages(),
                Detail::CreationDate(d) => doc.get_creation_date().map_or(false, |date| date == d),
            }
        }

        fn find_case_for_doc<'a>(
            &'a self,
            expected: bool,
            doc: &lopdf::Document,
        ) -> Option<Case<'a>> {
            if self.eval_doc(doc) == expected {
                let product = self.product_for_doc(doc);
                Some(Case::new(Some(self), false).add_product(product))
            } else {
                None
            }
        }

        fn product_for_doc(&self, doc: &lopdf::Document) -> Product {
            match self.d {
                Detail::PageCount(_) => Product::new(
                    "actual page count",
                    format!("{} page(s)", doc.get_num_pages()),
                ),
                Detail::CreationDate(_) => Product::new(
                    "actual creation date",
                    format!("{:?}", doc.get_creation_date()),
                ),
            }
        }
    }

    impl Details for lopdf::Document {
        fn get_creation_date(self: &Self) -> Option<DateTime<Utc>> {
            fn get_from_trailer<'a>(
                doc: &'a lopdf::Document,
                key: &[u8],
            ) -> lopdf::Result<&'a lopdf::Object> {
                let id = doc.trailer.get(b"Info")?.as_reference()?;
                doc.get_object(id)?.as_dict()?.get(key)
            }

            match get_from_trailer(self, b"CreationDate") {
                Ok(obj) => obj.as_datetime().map(|date| date.with_timezone(&Utc)),
                Err(_) => None,
            }
        }

        fn get_num_pages(self: &Self) -> usize {
            self.get_pages().len()
        }
    }

    impl Predicate<[u8]> for DetailPredicate<PdfPredicate> {
        fn eval(&self, data: &[u8]) -> bool {
            match lopdf::Document::load_mem(data) {
                Ok(doc) => self.eval_doc(&doc),
                _ => false,
            }
        }

        fn find_case<'a>(&'a self, expected: bool, data: &[u8]) -> Option<Case<'a>> {
            match lopdf::Document::load_mem(data) {
                Ok(doc) => self.find_case_for_doc(expected, &doc),
                Err(e) => Some(Case::new(Some(self), false).add_product(Product::new("Error", e))),
            }
        }
    }

    impl PredicateReflection for DetailPredicate<PdfPredicate> {
        fn children<'a>(&'a self) -> Box<dyn Iterator<Item = Child<'a>> + 'a> {
            let params = vec![Child::new("predicate", &self.p)];
            Box::new(params.into_iter())
        }
    }
{'fraction_non_alphanumeric': 0.1399213724088635, 'fraction_numerical': 0.0025017869907076485, 'mean_word_length': 1.6932146294513957, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 122, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4646443', 'n_tokens_mistral': 3478, 'n_tokens_neox': 3333, 'n_words': 675}