<commit_before>﻿module Yard.Core.Checkers

open Yard.Core.IL.Production
open System.Collections.Generic
open System.Linq

let private startRulesCount (def:Yard.Core.IL.Definition.t<_,_>) =
    def.grammar.Count(fun r -> r._public)

let IsStartRuleExists (def:Yard.Core.IL.Definition.t<_,_>) =
    startRulesCount def > 0

let IsSingleStartRule (def:Yard.Core.IL.Definition.t<_,_>) =
    startRulesCount def = 1

let IsChomskyNormalForm (def:Yard.Core.IL.Definition.t<_,_>) =
    def.grammar.All 
        (fun r ->
            match r.body with
            | PSeq([{rule = PToken(_);omit =_ ;binding =_;checker = _}],_)
            | PSeq([{rule = PRef(_);omit =_ ;binding =_;checker = _}
                   ;{rule = PRef(_);omit =_ ;binding =_;checker = _}],_) -> true 
            | _ -> false)

let GetUndeclaredNonterminalsList (def:Yard.Core.IL.Definition.t<_,_>) =
    let declaredRules = def.grammar |> List.map (fun r -> r.name) 
    let undeclaredRules = new HashSet<_>()
    let addUndeclaredRule name additionRules = 
        let name = (fst name).ToString()
        if (List.tryFind ( (=) name) (declaredRules) ).IsNone
        && (Seq.tryFind ( (=) name) (additionRules) ).IsNone then
            undeclaredRules.Add( name ) |> ignore

    let rec getUndeclaredRules (additionRules, body) =
        let getUndeclaredRulesCurried body = getUndeclaredRules (additionRules, body)
        match body with
            |PRef (name,_) -> addUndeclaredRule name additionRules
            |PMetaRef (name,_,exprList) ->  addUndeclaredRule name additionRules
                                            exprList |> List.iter (fun r -> getUndeclaredRulesCurried r )
            |PSeq (exprList,_) -> exprList |> List.iter (fun r -> getUndeclaredRulesCurried r.rule)
            |PPerm exprList  -> exprList |> List.iter (fun r -> getUndeclaredRulesCurried r)
            |PRepet (expr,_,_)
            |PMany expr
            |PSome expr
            |POpt  expr -> getUndeclaredRulesCurried expr
            |PAlt (lExpr,rExpr) -> 
                getUndeclaredRulesCurried lExpr
                getUndeclaredRulesCurried rExpr
            | _ -> ()

    def.grammar 
    |> List.iter 
        (fun r -> 
            let additionRules = new HashSet<_>()
            r.metaArgs |> List.iter (fun i -> additionRules.Add((fst i).ToString()) |> ignore)
            getUndeclaredRules (additionRules, r.body))

    List.ofSeq undeclaredRules

(*
let IsUndeclaredNonterminalsExists (def:Yard.Core.IL.Definition.t<_,_>) =
    let declaredRules = def.grammar |> List.map (fun r -> r.name) 
    let searchedRules = new HashSet<string>()
    let rec getUndeclaredRules (body:Yard.Core.IL.Production.t<_,_>) =
        match body with
            |PRef (name,_)
            |PMetaRef (name,_,_) -> let name = (fst name).ToString()
                                    if isNotViewedRule name then
                                        searchedRules.Add( name ) |> ignore
            |PSeq (exprList,_) -> exprList |> List.iter (fun r -> getUndeclaredRules r.rule )
            |PPerm (exprList) -> exprList |> List.iter (fun r -> getUndeclaredRules r )
            |PRepet (expr,_,_)
            |PMany(expr)
            |PSome (expr)
            |POpt (expr) -> getUndeclaredRules expr
            |PAlt (lExpr,rExpr) -> getUndeclaredRules lExpr<commit_msg>Revert "Code formatted. Unnecessary spaces and braces are removed."<commit_after>﻿module Yard.Core.Checkers

open Yard.Core.IL.Production
open System.Collections.Generic
open System.Linq

let private startRulesCount (def:Yard.Core.IL.Definition.t<_,_>) =
    def.grammar.Count(fun r -> r._public)

let IsStartRuleExists (def:Yard.Core.IL.Definition.t<_,_>) =
    startRulesCount def > 0

let IsSingleStartRule (def:Yard.Core.IL.Definition.t<_,_>) =
    startRulesCount def = 1

let IsChomskyNormalForm (def:Yard.Core.IL.Definition.t<_,_>) =
    def.grammar.All 
        (fun r ->
            match r.body with
            | PSeq([{rule = PToken(_);omit =_ ;binding =_;checker = _}],_)
            | PSeq([{rule = PRef(_);omit =_ ;binding =_;checker = _}
                   ;{rule = PRef(_);omit =_ ;binding =_;checker = _}],_) -> true 
            | _ -> false)

let GetUndeclaredNonterminalsList(def:Yard.Core.IL.Definition.t<_,_>) =
    let declaredRules = def.grammar |> List.map (fun r -> r.name) 
    let undeclaredRules = new HashSet<_>()
    let addUndeclaredRule name additionRules = 
        let name = (fst name).ToString()
        if (List.tryFind ( (=) name) (declaredRules) ).IsNone
        && (Seq.tryFind ( (=) name) (additionRules) ).IsNone then
            undeclaredRules.Add( name ) |> ignore

    let rec getUndeclaredRules (additionRules, body) =
        let getUndeclaredRulesCurried body = getUndeclaredRules (additionRules, body)
        match body with
            |PRef (name,_) ->   let name = (fst name).ToString()
                                if (List.tryFind ( (=) name) (declaredRules) ).IsNone
                                && (Seq.tryFind ( (=) name) (additionRules) ).IsNone then
                                    undeclaredRules.Add( name ) |> ignore
            |PSeq (exprList,_) -> exprList |> List.iter (fun r -> getUndeclaredRulesCurried r.rule )
            |PPerm (exprList) -> exprList |> List.iter (fun r -> getUndeclaredRulesCurried r )
            |PRepet (expr,_,_)
            |PMany(expr)
            |PSome (expr)
            |POpt (expr) -> getUndeclaredRulesCurried expr
            |PAlt (lExpr,rExpr) -> getUndeclaredRulesCurried lExpr
                                   getUndeclaredRulesCurried rExpr
            | _ -> ()

    def.grammar |> List.iter (fun r -> 
                                let additionRules = new HashSet<string>()
                                r.metaArgs |> List.iter (fun i -> additionRules.Add( (fst i).ToString() ) |> ignore )
                                getUndeclaredRules (additionRules, r.body)
                                )

    List.ofSeq undeclaredRules

(*
let IsUndeclaredNonterminalsExists (def:Yard.Core.IL.Definition.t<_,_>) =
    let declaredRules = def.grammar |> List.map (fun r -> r.name) 
    let searchedRules = new HashSet<string>()
    let rec getUndeclaredRules (body:Yard.Core.IL.Production.t<_,_>) =
        match body with
            |PRef (name,_)
            |PMetaRef (name,_,_) -> let name = (fst name).ToString()
                                    if isNotViewedRule name then
                                        searchedRules.Add( name ) |> ignore
            |PSeq (exprList,_) -> exprList |> List.iter (fun r -> getUndeclaredRules r.rule )
            |PPerm (exprList) -> exprList |> List.iter (fun r -> getUndeclaredRules r )
            |PRepet (expr,_,_)
            |PMany(expr)
            |PSome (expr)
            |POpt (expr) -> getUndeclaredRules expr
            |PAlt (lExpr,rExpr) -> getUndeclaredRules lExpr
{'fraction_non_alphanumeric': 0.12148856067187952, 'fraction_numerical': 0.0005792064871126556, 'mean_word_length': 2.072508896797153, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 97, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2630985', 'n_tokens_mistral': 2217, 'n_tokens_neox': 2104, 'n_words': 473}