<commit_before>use nom::{self,Needed,is_digit};
use nom::IResult::*;
use nom::Err::*;

use helper::*;
use super::{Date,Time,DateTime};

named!(take_4_digits, take_n_filter!(4, is_digit));
named!(take_2_digits, take_n_filter!(2, is_digit));

named!(sign, alt!(tag!("+") | tag!("-")));
named!(positive_year  <&[u8], i32>, map!(call!(take_4_digits), buf_to_i32));
named!(pub year <&[u8], i32>, chain!(
        pref: opt!(sign) ~
        y: positive_year
        ,
        || {
            match pref {
                Some(b"-") => -y,
                _ => y
            }
        }));

named!(pub month <&[u8], u32>, map!(call!(take_2_digits), buf_to_u32));
named!(pub day   <&[u8], u32>, map!(call!(take_2_digits), buf_to_u32));

named!(pub date <&[u8], Date>, chain!(
        y: year ~ tag!("-") ~ m: month ~ tag!("-") ~ d: day
        , || { Date{ year: y, month: m, day: d }
        }
        ));

named!(pub hour   <&[u8], u32>, map!(call!(take_2_digits), buf_to_u32));
named!(pub minute <&[u8], u32>, map!(call!(take_2_digits), buf_to_u32));
named!(pub second <&[u8], u32>, map!(call!(take_2_digits), buf_to_u32));

named!(pub time <&[u8], Time>, chain!(
        h: hour ~
        tag!(":") ~
        m: minute ~
        s: empty_or!(chain!(
                tag!(":") ~ s:second , || { s }))
        ,
        || { Time{ hour: h,
                   minute: m,
                   second: s.unwrap_or(0),
                   tz_offset: 0 }
           }
        ));

named!(timezone_hour <&[u8], i32>, chain!(
        s: sign ~
        h: hour ~
        empty_or!(
            chain!(
                tag!(":")? ~ m: minute , || { m }
            ))
        ,
        || {
            match s {
                b"-" => -(h as i32),
                _ => h as i32
            }
        }));

named!(tag_z, tag!("Z"));
named!(timezone_utc <&[u8], i32>, map!(tag_z, |_| 0));
named!(timezone <&[u8], i32>, alt!(timezone_utc | timezone_hour));

named!(pub datetime <&[u8], DateTime>, chain!(
        d: date ~
        tag!("T") ~
        t: time ~
        tzo: empty_or!(call!(timezone))
        ,
        || {
            DateTime{
                date: d,
                time: t.set_tz(tzo.unwrap_or(0)),
            }
        }
        ));
<commit_msg>Map sign to numeric<commit_after>use nom::{self,Needed,is_digit};
use nom::IResult::*;
use nom::Err::*;

use helper::*;
use super::{Date,Time,DateTime};

named!(take_4_digits, take_n_filter!(4, is_digit));
named!(take_2_digits, take_n_filter!(2, is_digit));

named!(sign, alt!(tag!("+") | tag!("-")));
named!(numeric_sign <&[u8], i32>, map!(sign, |s: &[u8]| {
    match s {
        b"-" => -1,
        _ => 1,
    }
}));

named!(positive_year  <&[u8], i32>, map!(call!(take_4_digits), buf_to_i32));
named!(pub year <&[u8], i32>, chain!(
        pref: opt!(numeric_sign) ~
        y: positive_year
        ,
        || {
            pref.unwrap_or(1) * y
        }));

named!(pub month <&[u8], u32>, map!(call!(take_2_digits), buf_to_u32));
named!(pub day   <&[u8], u32>, map!(call!(take_2_digits), buf_to_u32));

named!(pub date <&[u8], Date>, chain!(
        y: year ~ tag!("-") ~ m: month ~ tag!("-") ~ d: day
        , || { Date{ year: y, month: m, day: d }
        }
        ));

named!(pub hour   <&[u8], u32>, map!(call!(take_2_digits), buf_to_u32));
named!(pub minute <&[u8], u32>, map!(call!(take_2_digits), buf_to_u32));
named!(pub second <&[u8], u32>, map!(call!(take_2_digits), buf_to_u32));

named!(pub time <&[u8], Time>, chain!(
        h: hour ~
        tag!(":") ~
        m: minute ~
        s: empty_or!(chain!(
                tag!(":") ~ s:second , || { s }))
        ,
        || { Time{ hour: h,
                   minute: m,
                   second: s.unwrap_or(0),
                   tz_offset: 0 }
           }
        ));

named!(timezone_hour <&[u8], i32>, chain!(
        s: numeric_sign ~
        h: hour ~
        empty_or!(
            chain!(
                tag!(":")? ~ m: minute , || { m }
            ))
        ,
        || { s * (h as i32) }
        ));

named!(tag_z, tag!("Z"));
named!(timezone_utc <&[u8], i32>, map!(tag_z, |_| 0));
named!(timezone <&[u8], i32>, alt!(timezone_utc | timezone_hour));

named!(pub datetime <&[u8], DateTime>, chain!(
        d: date ~
        tag!("T") ~
        t: time ~
        tzo: empty_or!(call!(timezone))
        ,
        || {
            DateTime{
                date: d,
                time: t.set_tz(tzo.unwrap_or(0)),
            }
        }
        ));

{'fraction_non_alphanumeric': 0.21322056532383707, 'fraction_numerical': 0.02915646561317605, 'mean_word_length': 1.9979986657771849, 'pattern_counts': {'":': 6, '<': 30, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1912172', 'n_tokens_mistral': 1722, 'n_tokens_neox': 1673, 'n_words': 334}