<commit_before>extern crate chrono;
extern crate clap;
extern crate csv;
#[macro_use]
extern crate lazy_static;
extern crate regex;
extern crate safe_unwrap;
extern crate try_from;

use chrono::{NaiveDate, NaiveDateTime, NaiveTime};
use clap::{App, Arg, SubCommand};
use std::{io, path};
use regex::Regex;
use safe_unwrap::SafeUnwrap;
use try_from::TryFrom;

lazy_static! {
    static ref IDENT_UNDERSCORE_RE: Regex = Regex::new(
        r"^[a-z][a-z0-9_]*$"
    ).safe_unwrap("built-in Regex is broken. Please file a bug");
}

lazy_static! {
    static ref ENUM_EXPR_RE: Regex = Regex::new(
        r"^ENUM.*\(((?:[A-Z][A-Z0-9]*,?)*)\)$"
    ).safe_unwrap("built-in Regex is broken. Please file a bug");
}

lazy_static! {
    static ref CONSTRAINT_RE: Regex = Regex::new(
        r"^(:?(?:NULLABLE|UNIQUE),?)*$"
    ).safe_unwrap("built-in Regex is broken. Please file a bug");
}

lazy_static! {
    static ref DECIMAL_RE: Regex = Regex::new(
        r"^\d+(?:\.\d+)?$"
    ).safe_unwrap("built-in Regex is broken. Please file a bug");
}

lazy_static! {
    static ref DATE_RE: Regex = Regex::new(
        r"^(\d{4})(\d{2})(\d{2})$"
    ).safe_unwrap("built-in Regex is broken. Please file a bug");
}

lazy_static! {
    static ref DATETIME_RE: Regex = Regex::new(
        r"^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})$"
    ).safe_unwrap("built-in Regex is broken. Please file a bug");
}

lazy_static! {
    static ref TIME_RE: Regex = Regex::new(
        r"^(\d{2})(\d{2})(\d{2})$"
    ).safe_unwrap("built-in Regex is broken. Please file a bug");
}

lazy_static! {
    // `tablename_date_schema-schemaversion_csvxversion.csvx`
    static ref FN_RE: Regex = Regex::new(
        r"^([a-z][a-z0-9-]*)_(\d{4})(\d{2})(\d{2})_([a-z][a-z0-9-]*)_(\d+).csv$"
    ).expect("built-in Regex is broken. Please file a bug");
}

#[derive(Clone, Debug, Eq, PartialEq)]
struct CsvxMetadata {
    pub table_name: String,
    pub date: NaiveDate,
    pub schema: String,
    pub csvx_version: usize,
}

impl CsvxMetadata {
    fn is_schema(&self) -> bool {
        self.schema == "csvx-schema"
    }
}

#[derive(Debug)]
enum SchemaLoadError {
    Csv(csv::Error),
    MissingHeader,
    BadHeader,
    BadIdentifier(usize, String),
    BadType(usize, ColumnTypeError),
    BadConstraints(usize, ColumnConstraintsError),
}

#[derive(Debug)]
enum ValidationError {
    Csv(csv::Error),
    MissingHeaders,
    HeaderMismatch(usize, String),
    RowLengthMismatch(usize),
    ValueError(usize, usize, ValueError),
}

impl From<csv::Error> for ValidationError {
    fn from(e: csv::Error) -> ValidationError {
        ValidationError::Csv(e)
    }
}

#[derive(Clone, Debug)]
enum ColumnTypeError {
    UnknownType,
}

#[derive(Clone, Debug)]
enum ColumnType {
    String,
    Bool,
    Integer,
    Enum(Vec<String>),
    Decimal,
    Date,
    DateTime,
    Time,
}

#[derive(Clone, Debug)]
struct ColumnConstraints {
    nullable: bool,
    unique: bool,
}

impl Default for ColumnConstraints {
    fn default() -> ColumnConstraints {
        ColumnConstraints {
            nullable: false,
            unique: false,
        }
    }
}

#[derive(Clone, Debug)]
enum ColumnConstraintsError {
    MalformedConstraint,
    UnknownConstraint(String),
}

impl<S> TryFrom<S> for ColumnConstraints
    where S: AsRef<str>
{
    type Err = ColumnConstraintsError;

    fn try_from(s: S) -> Result<ColumnConstraints, Self::Err> {
        if !CONSTRAINT_RE.is_match(s.as_ref()) {
            return Err(ColumnConstraintsError::MalformedConstraint);
        }

        let mut ccs = ColumnConstraints::default();

        if s.as_ref() == "" {
            return Ok(ccs);
        }

        for fragment in s.as_ref().split(',') {
            match fragment.as_ref() {
                "NULLABLE" => {
                    ccs.nullable = true;
                }
                "UNIQUE" => {
                    ccs.unique = true;
                }
                _ => return Err(ColumnConstraintsError::UnknownConstraint(s.as_ref().to_string())),
            }

        }

        Ok(ccs)
    }
}

impl<S> TryFrom<S> for ColumnType
    where S: AsRef<str>
{
    type Err = ColumnTypeError;

    fn try_from(s: S) -> Result<ColumnType, Self::Err> {
        match s.as_ref() {
            "STRING" => Ok(ColumnType::String),
            "BOOL" => Ok(ColumnType::Bool),
            "INTEGER" => Ok(ColumnType::Integer),
            "DECIMAL" => Ok(ColumnType::Decimal),
            "DATE" => Ok(ColumnType::Date),
            "DATETIME" => Ok(ColumnType::DateTime),
            "TIME" => Ok(ColumnType::Time),
            _ if ENUM_EXPR_RE.is_match(s.as_ref()) => {
                let variants: Vec<_> = ENUM_EXPR_RE
                    .captures(s.as_ref())
                    .safe_unwrap("match already exists")
                    .get(1)
                    .safe_unwrap("group 1 exists in regex")
                    .as_str()
                    .split(',')
                    .map(|s| s.to_owned())
                    .collect();

                Ok(ColumnType::Enum(variants))
            }
            _ => return Err(ColumnTypeError::UnknownType),

        }
    }
}

#[derive(Clone, Debug)]
struct CsvxColumnType {
    id: String,
    ty: ColumnType,
    constraints: ColumnConstraints,
    description: String,
}

#[derive(Debug)]
enum Value {
    String(String),
    Bool(bool),
    Integer(i64),
    Enum(String),
    Decimal(String),
    Date(NaiveDate),
    DateTime(NaiveDateTime),
    Time(NaiveTime),
}

#[derive(Debug)]
enum ValueError {
    NonNullable,
    InvalidBool,
    InvalidInt,
    InvalidEnum,
    InvalidDecimal,
    InvalidDate,
    InvalidDateTime,
    InvalidTime,
}

impl From<std::num::ParseIntError> for ValueError {
    fn from(_: std::num::ParseIntError) -> ValueError {
        ValueError::InvalidInt
    }
}

impl CsvxColumnType {
    fn validate_value<S: AsRef<str>>(&self, s: &S) -> Result<Option<Value>, ValueError> {
        // FIXME: check UNIQUE

        // null check
        if s.as_ref() == "" {
            if self.constraints.nullable {
                return Ok(None);
            } else {
                return Err(ValueError::NonNullable);
            }
        }

        match self.ty {
            ColumnType::String => Ok(Some(Value::String(s.as_ref().to_string()))),
            ColumnType::Bool => {
                match s.as_ref() {
                    "TRUE" => Ok(Some(Value::Bool(true))),
                    "FALSE" => Ok(Some(Value::Bool(false))),
                    _ => Err(ValueError::InvalidBool),
                }
            }
            ColumnType::Integer => {
                // FIXME: check for leading zeros
                Ok(Some(Value::Integer(s.as_ref().parse()?)))
            }
            ColumnType::Enum(ref variants) => {
                let v = s.as_ref().to_owned();
                if variants.contains(&v) {
                    Ok(Some(Value::Enum(v)))
                } else {
                    Err(ValueError::InvalidEnum)
                }
            }
            ColumnType::Decimal => {
                if !DECIMAL_RE.is_match(s.as_ref()) {
                    Ok(Some(Value::Decimal(s.as_ref().to_owned())))
                } else {
                    Err(ValueError::InvalidDecimal)
                }
            }
            ColumnType::Date => {
                match DATE_RE.captures(s.as_ref()) {
                    Some(ref c) => {
                        Ok(Some(Value::Date(NaiveDate::from_ymd_opt(cap(c, 1),
                                                                    cap(c, 2),
                                                                    cap(c, 3))
                                                    .ok_or(ValueError::InvalidDate)?)))
                    }
                    None => Err(ValueError::InvalidDate),<commit_msg>Fixed logic-inversion error in DECIMAL_RE application.<commit_after>extern crate chrono;
extern crate clap;
extern crate csv;
#[macro_use]
extern crate lazy_static;
extern crate regex;
extern crate safe_unwrap;
extern crate try_from;

use chrono::{NaiveDate, NaiveDateTime, NaiveTime};
use clap::{App, Arg, SubCommand};
use std::{io, path};
use regex::Regex;
use safe_unwrap::SafeUnwrap;
use try_from::TryFrom;

lazy_static! {
    static ref IDENT_UNDERSCORE_RE: Regex = Regex::new(
        r"^[a-z][a-z0-9_]*$"
    ).safe_unwrap("built-in Regex is broken. Please file a bug");
}

lazy_static! {
    static ref ENUM_EXPR_RE: Regex = Regex::new(
        r"^ENUM.*\(((?:[A-Z][A-Z0-9]*,?)*)\)$"
    ).safe_unwrap("built-in Regex is broken. Please file a bug");
}

lazy_static! {
    static ref CONSTRAINT_RE: Regex = Regex::new(
        r"^(:?(?:NULLABLE|UNIQUE),?)*$"
    ).safe_unwrap("built-in Regex is broken. Please file a bug");
}

lazy_static! {
    static ref DECIMAL_RE: Regex = Regex::new(
        r"^\d+(?:\.\d+)?$"
    ).safe_unwrap("built-in Regex is broken. Please file a bug");
}

lazy_static! {
    static ref DATE_RE: Regex = Regex::new(
        r"^(\d{4})(\d{2})(\d{2})$"
    ).safe_unwrap("built-in Regex is broken. Please file a bug");
}

lazy_static! {
    static ref DATETIME_RE: Regex = Regex::new(
        r"^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})$"
    ).safe_unwrap("built-in Regex is broken. Please file a bug");
}

lazy_static! {
    static ref TIME_RE: Regex = Regex::new(
        r"^(\d{2})(\d{2})(\d{2})$"
    ).safe_unwrap("built-in Regex is broken. Please file a bug");
}

lazy_static! {
    // `tablename_date_schema-schemaversion_csvxversion.csvx`
    static ref FN_RE: Regex = Regex::new(
        r"^([a-z][a-z0-9-]*)_(\d{4})(\d{2})(\d{2})_([a-z][a-z0-9-]*)_(\d+).csv$"
    ).expect("built-in Regex is broken. Please file a bug");
}

#[derive(Clone, Debug, Eq, PartialEq)]
struct CsvxMetadata {
    pub table_name: String,
    pub date: NaiveDate,
    pub schema: String,
    pub csvx_version: usize,
}

impl CsvxMetadata {
    fn is_schema(&self) -> bool {
        self.schema == "csvx-schema"
    }
}

#[derive(Debug)]
enum SchemaLoadError {
    Csv(csv::Error),
    MissingHeader,
    BadHeader,
    BadIdentifier(usize, String),
    BadType(usize, ColumnTypeError),
    BadConstraints(usize, ColumnConstraintsError),
}

#[derive(Debug)]
enum ValidationError {
    Csv(csv::Error),
    MissingHeaders,
    HeaderMismatch(usize, String),
    RowLengthMismatch(usize),
    ValueError(usize, usize, ValueError),
}

impl From<csv::Error> for ValidationError {
    fn from(e: csv::Error) -> ValidationError {
        ValidationError::Csv(e)
    }
}

#[derive(Clone, Debug)]
enum ColumnTypeError {
    UnknownType,
}

#[derive(Clone, Debug)]
enum ColumnType {
    String,
    Bool,
    Integer,
    Enum(Vec<String>),
    Decimal,
    Date,
    DateTime,
    Time,
}

#[derive(Clone, Debug)]
struct ColumnConstraints {
    nullable: bool,
    unique: bool,
}

impl Default for ColumnConstraints {
    fn default() -> ColumnConstraints {
        ColumnConstraints {
            nullable: false,
            unique: false,
        }
    }
}

#[derive(Clone, Debug)]
enum ColumnConstraintsError {
    MalformedConstraint,
    UnknownConstraint(String),
}

impl<S> TryFrom<S> for ColumnConstraints
    where S: AsRef<str>
{
    type Err = ColumnConstraintsError;

    fn try_from(s: S) -> Result<ColumnConstraints, Self::Err> {
        if !CONSTRAINT_RE.is_match(s.as_ref()) {
            return Err(ColumnConstraintsError::MalformedConstraint);
        }

        let mut ccs = ColumnConstraints::default();

        if s.as_ref() == "" {
            return Ok(ccs);
        }

        for fragment in s.as_ref().split(',') {
            match fragment.as_ref() {
                "NULLABLE" => {
                    ccs.nullable = true;
                }
                "UNIQUE" => {
                    ccs.unique = true;
                }
                _ => return Err(ColumnConstraintsError::UnknownConstraint(s.as_ref().to_string())),
            }

        }

        Ok(ccs)
    }
}

impl<S> TryFrom<S> for ColumnType
    where S: AsRef<str>
{
    type Err = ColumnTypeError;

    fn try_from(s: S) -> Result<ColumnType, Self::Err> {
        match s.as_ref() {
            "STRING" => Ok(ColumnType::String),
            "BOOL" => Ok(ColumnType::Bool),
            "INTEGER" => Ok(ColumnType::Integer),
            "DECIMAL" => Ok(ColumnType::Decimal),
            "DATE" => Ok(ColumnType::Date),
            "DATETIME" => Ok(ColumnType::DateTime),
            "TIME" => Ok(ColumnType::Time),
            _ if ENUM_EXPR_RE.is_match(s.as_ref()) => {
                let variants: Vec<_> = ENUM_EXPR_RE
                    .captures(s.as_ref())
                    .safe_unwrap("match already exists")
                    .get(1)
                    .safe_unwrap("group 1 exists in regex")
                    .as_str()
                    .split(',')
                    .map(|s| s.to_owned())
                    .collect();

                Ok(ColumnType::Enum(variants))
            }
            _ => return Err(ColumnTypeError::UnknownType),

        }
    }
}

#[derive(Clone, Debug)]
struct CsvxColumnType {
    id: String,
    ty: ColumnType,
    constraints: ColumnConstraints,
    description: String,
}

#[derive(Debug)]
enum Value {
    String(String),
    Bool(bool),
    Integer(i64),
    Enum(String),
    Decimal(String),
    Date(NaiveDate),
    DateTime(NaiveDateTime),
    Time(NaiveTime),
}

#[derive(Debug)]
enum ValueError {
    NonNullable,
    InvalidBool,
    InvalidInt,
    InvalidEnum,
    InvalidDecimal,
    InvalidDate,
    InvalidDateTime,
    InvalidTime,
}

impl From<std::num::ParseIntError> for ValueError {
    fn from(_: std::num::ParseIntError) -> ValueError {
        ValueError::InvalidInt
    }
}

impl CsvxColumnType {
    fn validate_value<S: AsRef<str>>(&self, s: &S) -> Result<Option<Value>, ValueError> {
        // FIXME: check UNIQUE

        // null check
        if s.as_ref() == "" {
            if self.constraints.nullable {
                return Ok(None);
            } else {
                return Err(ValueError::NonNullable);
            }
        }

        match self.ty {
            ColumnType::String => Ok(Some(Value::String(s.as_ref().to_string()))),
            ColumnType::Bool => {
                match s.as_ref() {
                    "TRUE" => Ok(Some(Value::Bool(true))),
                    "FALSE" => Ok(Some(Value::Bool(false))),
                    _ => Err(ValueError::InvalidBool),
                }
            }
            ColumnType::Integer => {
                // FIXME: check for leading zeros
                Ok(Some(Value::Integer(s.as_ref().parse()?)))
            }
            ColumnType::Enum(ref variants) => {
                let v = s.as_ref().to_owned();
                if variants.contains(&v) {
                    Ok(Some(Value::Enum(v)))
                } else {
                    Err(ValueError::InvalidEnum)
                }
            }
            ColumnType::Decimal => {
                if DECIMAL_RE.is_match(s.as_ref()) {
                    Ok(Some(Value::Decimal(s.as_ref().to_owned())))
                } else {
                    Err(ValueError::InvalidDecimal)
                }
            }
            ColumnType::Date => {
                match DATE_RE.captures(s.as_ref()) {
                    Some(ref c) => {
                        Ok(Some(Value::Date(NaiveDate::from_ymd_opt(cap(c, 1),
                                                                    cap(c, 2),
                                                                    cap(c, 3))
                                                    .ok_or(ValueError::InvalidDate)?)))
                    }
                    None => Err(ValueError::InvalidDate),
{'fraction_non_alphanumeric': 0.14650127226463106, 'fraction_numerical': 0.003816793893129771, 'mean_word_length': 1.9662264150943396, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 95, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4070589', 'n_tokens_mistral': 5088, 'n_tokens_neox': 5007, 'n_words': 948}