<commit_before>﻿//  Module ReplaceInline replace nonterminals, from which only
//    Pref, PToken or PLiteral can be infered, whit their right sides.
//
//  Copyright 2011, 2012 <NAME>
//
//  This file is part of YaccConctructor.
//  This is war
//
//  YaccConstructor is free software:you can redistribute it and/or modify
//  it under the terms of the GNU General Public License as published by
//  the Free Software Foundation, either version 3 of the License, or
//  (at your option) any later version.
//
//  This program is distributed in the hope that it will be useful,
//  but WITHOUT ANY WARRANTY; without even the implied warranty of
//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//  GNU General Public License for more details.
//
//  You should have received a copy of the GNU General Public License
//  along with this program.  If not, see <http://www.gnu.org/licenses/>.

module Yard.Core.Convertions.ToCNF

open System.IO
open Yard.Core
open Yard.Core.IL
open Yard.Core.IL.Production
open System.Collections.Generic
open Yard.Core.IL.Rule

//--Функция для удаления эпсилон-правил------------------------------------------------------------

let deleteEpsRule (ruleList: Rule.t<_,_> list) = 
    
    //--Генерация перестановок---------------------------------------------------------------------
        
    let genPermutation N = 
        let genList = ref [[]]
        let addInList list = 
            genList := list @ !genList
            list
        let startList = [for i in [1 .. N] -> [i]]
        genList := startList
        let rec iter listList = 
            if List.length listList <> 1 then
                listList |> List.collect 
                    (fun i -> [List.max i + 1 .. N] |> List.collect (fun j -> [ i @ [j] ])) 
                |> addInList 
                |> iter                  
        iter startList
        !genList
        
    //--Находим все эпсилон-правила----------------------------------------------------------------

    let epsList = 
        ruleList |> List.collect
            (fun rule -> 
                match rule.body with
                |PSeq(elements, actionCode, lbl) when elements.IsEmpty -> [rule.name]
                |x -> []
            )

    //--Функция для проверки вхождения эпсилон-правила---------------------------------------------

    let isEps s = 
        epsList |> List.collect (fun eps -> if s = eps then [eps] else [])
    
    //--Список эпсилон-правил входящих в данное правило--------------------------------------------     

    let rec epsInRule elements = 
        elements |> List.collect
                    (fun elem ->
                        match elem.rule with
                        |PSeq(e, a, l) -> epsInRule e
                        |PRef(t, _) -> isEps (fst t)
                        |x -> []
                    )
                        
    //--Функция для добавления нового правила------------------------------------------------------

    let newRule (rule: Rule.t<_, _>) (epsRef: list<string>) = 
        if not epsRef.IsEmpty then
            let numberEpsRef = genPermutation epsRef.Length 
            let numberBody = 
                let i = ref 0
                match rule.body with
                |PSeq(elements, _, _) -> 
                    PSeq(
                        elements |> List.collect
                            (fun elem ->
                                match elem.rule with
                                |PRef(t, _) when fst t |> isEps |> List.isEmpty |> not-> 
                                    incr i
                                    [{
                                        omit = elem.omit
                                        rule = PRef(( (!i).ToString(), (0, 0)), None)
                                        binding = elem.binding
                                        checker = elem.checker
                                    }]
                                |x -> [elem]
                            ),
                            (match rule.body with PSeq(e, a, l) -> a | x -> None),
                            (match rule.body with PSeq(e, a, l) -> l | x -> None))
                |x -> rule.body
            let addRule (numberRule: Rule.t<_, _>) eps =                
                [{
                    name=numberRule.name
                    args=numberRule.args
                    _public=numberRule._public
                    metaArgs=numberRule.metaArgs
                    body=PSeq(
                                (match numberRule.body with PSeq(e, a, l) -> e | x -> [])
                                |> List.collect 
                                    (fun elem ->
                                        match elem.rule with
                                        |PRef(t, _) when 
                                            eps
                                            |> List.map (fun e -> string e) 
                                            |> List.exists (fun e -> e = fst t) -> []
                                        |PRef(t, _) when 
                                            not (
                                                eps 
                                                |> List.map (fun e -> e.ToString()) 
                                                |> List.exists (fun e -> (e = (fst t)))) -> 
                                             [{
                                                omit = elem.omit
                                                rule = PRef((epsRef.Item(System.Convert.ToInt32(fst t) - 1).ToString(), (0, 0)), None)
                                                binding = elem.binding
                                                checker = elem.checker
                                            }]
                                        |x -> [elem]
                                    )
                                ,
                                (match numberRule.body with PSeq(e, a, l) -> a | x -> None), 
                                (match numberRule.body with PSeq(e, a, l) -> l | x -> None))
                }]
            let numberRule = 
                {
                    name=rule.name
                    args=rule.args
                    _public=rule._public
                    metaArgs=rule.metaArgs
                    body=numberBody
                }
            numberEpsRef |> List.collect
                (fun eps -> addRule numberRule eps)
        else []
            
            
    //--Добавляем новые правила--------------------------------------------------------------------
    
    ruleList |> List.collect
        (fun rule -> 
            match rule.body with
            |PSeq(elements, actionCode, lbl) when not elements.IsEmpty -> 
                newRule rule (epsInRule elements) @ [rule]
            |x -> []
        )

//--Функция для удаления цепных правил-------------------------------------------------------------
                
let deleteChainRule (ruleList: Rule.t<_, _> list) = 
    
    let rec newRule (mainRule: Rule.t<_, _>) name = 
        ruleList |> List.collect
            (fun rule ->

                let isOneRule rule = 
                    match rule.body with
                    |PSeq(elements, actionCode, lbl) 
                        when elements.Length = 1
                        && (match elements.Head.rule with PRef(t, _) -> true | x -> false) -> true
                    |x -> false<commit_msg>Refactoring of newRule function.<commit_after>﻿//  Module ReplaceInline replace nonterminals, from which only
//    Pref, PToken or PLiteral can be infered, whit their right sides.
//
//  Copyright 2011, 2012 <NAME>
//
//  This file is part of YaccConctructor.
//  This is war
//
//  YaccConstructor is free software:you can redistribute it and/or modify
//  it under the terms of the GNU General Public License as published by
//  the Free Software Foundation, either version 3 of the License, or
//  (at your option) any later version.
//
//  This program is distributed in the hope that it will be useful,
//  but WITHOUT ANY WARRANTY; without even the implied warranty of
//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//  GNU General Public License for more details.
//
//  You should have received a copy of the GNU General Public License
//  along with this program.  If not, see <http://www.gnu.org/licenses/>.

module Yard.Core.Convertions.ToCNF

open System.IO
open Yard.Core
open Yard.Core.IL
open Yard.Core.IL.Production
open System.Collections.Generic
open Yard.Core.IL.Rule

//--Функция для удаления эпсилон-правил------------------------------------------------------------

let deleteEpsRule (ruleList: Rule.t<_,_> list) = 
    
    //--Генерация перестановок---------------------------------------------------------------------
        
    let genPermutation N = 
        let genList = ref [[]]
        let addInList list = 
            genList := list @ !genList
            list
        let startList = [for i in [1 .. N] -> [i]]
        genList := startList
        let rec iter listList = 
            if List.length listList <> 1 then
                listList |> List.collect 
                    (fun i -> [List.max i + 1 .. N] |> List.collect (fun j -> [ i @ [j] ])) 
                |> addInList 
                |> iter                  
        iter startList
        !genList
        
    //--Находим все эпсилон-правила----------------------------------------------------------------

    let epsList = 
        ruleList |> List.collect
            (fun rule -> 
                match rule.body with
                |PSeq(elements, actionCode, lbl) when elements.IsEmpty -> [rule.name]
                |x -> []
            )

    //--Функция для проверки вхождения эпсилон-правила---------------------------------------------

    let isEps s = 
        epsList |> List.collect (fun eps -> if s = eps then [eps] else [])
    
    //--Список эпсилон-правил входящих в данное правило--------------------------------------------     

    let rec epsInRule elements = 
        elements |> List.collect
                    (fun elem ->
                        match elem.rule with
                        |PSeq(e, a, l) -> epsInRule e
                        |PRef(t, _) -> isEps (fst t)
                        |x -> []
                    )
                        
    //--Функция для добавления нового правила------------------------------------------------------

    let newRule (rule: Rule.t<_, _>) (epsRef: list<string>) =         
        if not epsRef.IsEmpty then
            let numberEpsRef = genPermutation epsRef.Length
            let ac,lbl = match rule.body with PSeq(e, a, l) -> a,l | x -> None,None
            let i = ref 0
            let newBody elements =
                elements 
                |> List.collect
                    (fun elem ->
                        match elem.rule with
                        |PRef(t, _) when fst t |> isEps |> List.isEmpty |> not-> 
                            incr i
                            [{
                                omit = elem.omit
                                rule = PRef((string !i, (0, 0)), None)
                                binding = elem.binding
                                checker = elem.checker
                            }]
                        |x -> [elem]
                    )
            let numberBody =                
                match rule.body with
                |PSeq(elements, _, _) -> 
                    PSeq(newBody elements, ac, lbl)
                |x -> rule.body
            let addRule (numberRule: Rule.t<_, _>) eps =
                let epsWithNameExists t = 
                    eps
                    |> List.map string
                    |> List.exists ((=) (fst t))
                let ac,lbl = match numberRule.body with PSeq(e, a, l) -> a,l | x -> None,None
                let newBody = 
                    match numberRule.body with PSeq(e, a, l) -> e | x -> []
                    |> List.collect 
                        (fun elem ->
                            match elem.rule with
                            |PRef(t,_) when epsWithNameExists t -> []
                            |PRef(t,_) when not <| epsWithNameExists t -> 
                                    [{
                                        omit = elem.omit
                                        rule = PRef((epsRef.[int (fst t) - 1], (0, 0)), None)
                                        binding = elem.binding
                                        checker = elem.checker
                                    }]
                            |x -> [elem])
                [{
                    name=numberRule.name
                    args=numberRule.args
                    _public=numberRule._public
                    metaArgs=numberRule.metaArgs
                    body=PSeq(newBody, ac, lbl)
                }]
            let numberRule = 
                {
                    name=rule.name
                    args=rule.args
                    _public=rule._public
                    metaArgs=rule.metaArgs
                    body=numberBody
                }
            numberEpsRef |> List.collect (addRule numberRule)
        else []
            
            
    //--Добавляем новые правила--------------------------------------------------------------------
    
    ruleList |> List.collect
        (fun rule -> 
            match rule.body with
            |PSeq(elements, actionCode, lbl) when not elements.IsEmpty -> 
                newRule rule (epsInRule elements) @ [rule]
            |x -> []
        )

//--Функция для удаления цепных правил-------------------------------------------------------------
                
let deleteChainRule (ruleList: Rule.t<_, _> list) = 
    
    let rec newRule (mainRule: Rule.t<_, _>) name = 
        ruleList |> List.collect
            (fun rule ->

                let isOneRule rule = 
                    match rule.body with
                    |PSeq(elements, actionCode, lbl) 
                        when elements.Length = 1
                        && (match elements.Head.rule with PRef(t, _) -> true | x -> false) -> true
                    |x -> false
{'fraction_non_alphanumeric': 0.14901960784313725, 'fraction_numerical': 0.0028011204481792717, 'mean_word_length': 0.8682626896912611, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 134, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 53, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '111446', 'n_tokens_mistral': 3573, 'n_tokens_neox': 3358, 'n_words': 1010}