<commit_before>//! frust query parser
//!
//! # Talkin' Syntax
//! For now the precendence of 'and' and 'or' is not implemented, nor are brackets supported.
//! This meens that if you want (f1 and f2) or f3, you have to express it as f3 or f1 and f2.
//! I know this is limitting and I'm going to improve this in the future...

use std::string::String;
use filter;
use filter_tree;
use query;

named!(komma, ws!(tag!(",")));
named!(comp_op<filter::CompOp>, alt!(
          tag!("<")  => { |_| filter::CompOp::Lower }  
        | tag!("<=") => { |_| filter::CompOp::LowerEqual }
        | tag!("==") => { |_| filter::CompOp::Equal }
        | tag!("not =") => { |_| filter::CompOp::Equal }
        | tag!(">=") => { |_| filter::CompOp::GreaterEqual }
        | tag!(">")  => { |_| filter::CompOp::Greater }
        | tag!("~")  => { |_| filter::CompOp::Like }
        | tag!("not ~")  => { |_| filter::CompOp::Unlike }
        )
    );

named!(logic_op<filter_tree::LogicOp>, 
       alt!(
        ws!(tag!("or")) =>  { |_| filter_tree::LogicOp::Or }
        | 
        ws!(tag!("and")) => { |_| filter_tree::LogicOp::And }
        )
);

named!(from_keyword, ws!(tag!("from")));
named!(where_keyword, ws!(tag!("where")));
named!(exec_keyword, ws!(tag!("exec")));
named!(attribute<filter::Attribute>, 
       do_parse!(
           tag: ws!(alt!( 
                      tag!("name")     => { |_| filter::Attribute::Name }
                    | tag!("size")     => { |_| filter::Attribute::Size }
                    | tag!("mtime")    => { |_| filter::Attribute::Mtime }
                    | tag!("ctime")    => { |_| filter::Attribute::Ctime }
                    | tag!("atime")    => { |_| filter::Attribute::Atime }
                    | tag!("filetype") => { |_| filter::Attribute::Filetype }
                    | tag!("mimetype") => { |_| filter::Attribute::Mimetype }
                    | tag!("inode")    => { |_| filter::Attribute::Inode }
                    | tag!("basename") => { |_| filter::Attribute::Basename }
                )
               ) >>
               (tag)
           )
       );

named!(directory<String>, 
       do_parse!(
           dir: re_bytes_find!("(/)?([^/\0,; ]+(/)?)+") >>
           (String::from_utf8_lossy(dir).into_owned())
       )
   );
named!(num_paramter, re_bytes_find!("-?[0-9]+(\\.[0-9]*)?[a-zA-Z]"));
named!(str_paramter, 
       do_parse!(
           tag!("'") >> 
           pattern: re_bytes_find!("[^']+") >>
           tag!("'") >>
           (pattern)
           )
       );
named!(date_parameter, re_bytes_find!("(([0-9]{4}-[0-9]{2}-[0-9]{2})? ?[0-9]{1,2}:[0-9]{2})|([0-9]{4}-[0-9]{2}-[0-9]{2})"));

named!(parameter, alt!(num_paramter | str_paramter | date_parameter));

named!(filter<filter_tree::FilterTuple>, 
       do_parse!(
           attr: attribute >>
           op  : ws!(comp_op)   >>
           param: ws!(parameter) >>
           (filter_tree::FilterTuple::new(attr, 
                                op, 
                                String::from_utf8_lossy(param).into_owned()<commit_msg>Bug fix: parse '/' as directory<commit_after>//! frust query parser
//!
//! # Talkin' Syntax
//! For now the precendence of 'and' and 'or' is not implemented, nor are brackets supported.
//! This meens that if you want (f1 and f2) or f3, you have to express it as f3 or f1 and f2.
//! I know this is limitting and I'm going to improve this in the future...

use std::string::String;
use filter;
use filter_tree;
use query;

named!(komma, ws!(tag!(",")));
named!(comp_op<filter::CompOp>, alt!(
          tag!("<")  => { |_| filter::CompOp::Lower }  
        | tag!("<=") => { |_| filter::CompOp::LowerEqual }
        | tag!("==") => { |_| filter::CompOp::Equal }
        | tag!("not =") => { |_| filter::CompOp::Equal }
        | tag!(">=") => { |_| filter::CompOp::GreaterEqual }
        | tag!(">")  => { |_| filter::CompOp::Greater }
        | tag!("~")  => { |_| filter::CompOp::Like }
        | tag!("not ~")  => { |_| filter::CompOp::Unlike }
        )
    );

named!(logic_op<filter_tree::LogicOp>, 
       alt!(
        ws!(tag!("or")) =>  { |_| filter_tree::LogicOp::Or }
        | 
        ws!(tag!("and")) => { |_| filter_tree::LogicOp::And }
        )
);

named!(from_keyword, ws!(tag!("from")));
named!(where_keyword, ws!(tag!("where")));
named!(exec_keyword, ws!(tag!("exec")));
named!(attribute<filter::Attribute>, 
       do_parse!(
           tag: ws!(alt!( 
                      tag!("name")     => { |_| filter::Attribute::Name }
                    | tag!("size")     => { |_| filter::Attribute::Size }
                    | tag!("mtime")    => { |_| filter::Attribute::Mtime }
                    | tag!("ctime")    => { |_| filter::Attribute::Ctime }
                    | tag!("atime")    => { |_| filter::Attribute::Atime }
                    | tag!("filetype") => { |_| filter::Attribute::Filetype }
                    | tag!("mimetype") => { |_| filter::Attribute::Mimetype }
                    | tag!("inode")    => { |_| filter::Attribute::Inode }
                    | tag!("basename") => { |_| filter::Attribute::Basename }
                )
               ) >>
               (tag)
           )
       );

named!(directory<String>, 
       do_parse!(
           dir: re_bytes_find!("/|(/)?([^/\0,; ]+(/)?)+") >>
           (String::from_utf8_lossy(dir).into_owned())
       )
   );
named!(num_paramter, re_bytes_find!("-?[0-9]+(\\.[0-9]*)?[a-zA-Z]"));
named!(str_paramter, 
       do_parse!(
           tag!("'") >> 
           pattern: re_bytes_find!("[^']+") >>
           tag!("'") >>
           (pattern)
           )
       );
named!(date_parameter, re_bytes_find!("(([0-9]{4}-[0-9]{2}-[0-9]{2})? ?[0-9]{1,2}:[0-9]{2})|([0-9]{4}-[0-9]{2}-[0-9]{2})"));

named!(parameter, alt!(num_paramter | str_paramter | date_parameter));

named!(filter<filter_tree::FilterTuple>, 
       do_parse!(
           attr: attribute >>
           op  : ws!(comp_op)   >>
           param: ws!(parameter) >>
           (filter_tree::FilterTuple::new(attr, 
                                op, 
                                String::from_utf8_lossy(param).into_owned()
{'fraction_non_alphanumeric': 0.22594278283485045, 'fraction_numerical': 0.01235370611183355, 'mean_word_length': 2.161870503597122, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5953007', 'n_tokens_mistral': 2095, 'n_tokens_neox': 2062, 'n_words': 307}