<commit_before>//    F# .  http://fsharp.org
//   .   "  F#".

namespace YC.BioGraph

type Nuc = | A | C | G | U

type Edge = int * int * Nuc * bool

type Graph = {
    countOfVertex: int;
    edges: Edge[];
}

module Parser =
    open AbstractAnalysis.Common
    open Yard.Frontends.YardFrontend.Main
    open Yard.Generators.GLL.AbstractParser
    open AbstractAnalysis.Common
    open Yard.Generators.GLL
    open Yard.Generators.Common.FinalGrammar
    open Yard.Generators.Common.InitialConvert
    open Yard.Generators.Common.ASTGLL
    open Yard.Generators.GLL.ParserCommon
    open System.Collections.Generic
    open Yard.Generators.GLL
    open Yard.Generators.Common

    type Token =
        | A
        | C
        | G
        | U
        | EOF

    let graphParse graph'text =
        let gd = DotParser.parse graph'text
        let n = gd.Nodes.Count
        let g = new ParserInputGraph<Token>([|0..n|], [|n|])

        for i in 0..n do
            ignore <| g.AddVertex i

        for i in 0..n - 1 do
            ignore <| g.AddEdge(new ParserEdge<Token>(i, n, EOF))
    
        for edge in gd.Edges do
            let (x, y) = edge.Key
            let (a, b) = ref 0, ref 0
            if System.Int32.TryParse(x, a) && System.Int32.TryParse(y, b) then
                match edge.Value.Head.["label"] with
                | "A" -> ignore <| g.AddEdge(new ParserEdge<Token>(!a, !b, A))
                | "C" -> ignore <| g.AddEdge(new ParserEdge<Token>(!a, !b, C))
                | "G" -> ignore <| g.AddEdge(new ParserEdge<Token>(!a, !b, G))
                | "U" -> ignore <| g.AddEdge(new ParserEdge<Token>(!a, !b, U))
                | _ -> ()
        g
        (*
        let genGraphNWithEof n edges =
            let g = new ParserInputGraph<Token>([|0..n|], [|n|])

            for i in 0..n do
                ignore <| g.AddVertex i
    
            for i in 0..n - 1 do
                ignore <| g.AddEdge(new ParserEdge<Token>(i, n, EOF))
    
            for s, t, tok in edges do
                ignore <| g.AddEdge(new ParserEdge<Token>(s, t, tok))
    
            g

        let graph: ParserInputGraph<Token> =
            genGraphNWithEof 3 [0, 1, U; 1, 2, C]
        *)
    
    let mutable indToString = fun i -> ""
    let mutable tokenToNumber = fun t -> 0
    let tokenData (t:Token): obj = null

    let grmParse parser_text = 
        let text = parser_text
        (*let text = @"
        [<Start>]
        s: a b | b c | d
        a: A
        b: C
        c: G
        d: U"*)
        let grm = ParseText text "file.yrd"
        let icg = initialConvert grm
        let fg = FinalGrammar(icg.grammar.[0].rules, true)

        tokenToNumber <- function
            | A -> fg.indexator.termToIndex "A"
            | C -> fg.indexator.termToIndex "C"
            | G -> fg.indexator.termToIndex "G"
            | U -> fg.indexator.termToIndex "U"
            | EOF -> fg.indexator.eofIndex

        let genLiteral (s:string) (i:int): Token option = None
    
        let isLiteral (t:Token): bool = false
        let isTerminal (t:Token): bool = true
        let getLiteralNames = []

        let td = (Table fg).result
        let table = new System.Collections.Generic.Dictionary<int, int[]>()
        for k in td.Keys do
            table.Add(k, td.[k].ToArray())

        let rulesArr = Array.zeroCreate fg.rules.rulesCount
        for i = 0 to fg.rules.rulesCount-1 do
            rulesArr.[i] <- fg.rules.rightSide i

        let totalRulesLength = rulesArr |> Array.sumBy (fun x -> x.Length)
        let rules = Array.zeroCreate totalRulesLength
        let rulesStart = Array.zeroCreate <| fg.rules.rulesCount + 1
        let mutable cur = 0
        for i = 0 to fg.rules.rulesCount-1 do
            rulesStart.[i] <- cur
            for j = 0 to rulesArr.[i].Length-1 do
                rules.[cur] <- rulesArr.[i].[j]
                cur <- cur + 1
        rulesStart.[fg.rules.rulesCount] <- cur

        let acceptEmptyInput = true
        let numIsTerminal (i:int): bool = fg.indexator.termsStart <= i && i <= fg.indexator.termsEnd
        let numIsNonTerminal (i:int): bool = fg.indexator.isNonTerm i
        let numIsLiteral (i:int): bool = fg.indexator.literalsStart <= i && i <= fg.indexator.literalsEnd

        let numToString (n:int):string =
            if numIsTerminal n then
                fg.indexator.indexToTerm n
            elif numIsNonTerminal n then
                fg.indexator.indexToNonTerm n
            elif numIsLiteral n then
                fg.indexator.indexToLiteral n
            else string n
    
        indToString <- numToString

        let inline packRulePosition rule position = (int rule <<< 16) ||| int position

        let slots = new List<_>()
        slots.Add(packRulePosition -1 -1, 0)
        for i = 0 to fg.rules.rulesCount - 1 do
            let currentRightSide = fg.rules.rightSide i
            for j = 0 to currentRightSide.Length - 1 do
                if fg.indexator.isNonTerm currentRightSide.[j] then
                    let key = packRulePosition i (j + 1)
                    slots.Add(key, slots.Count)

        let parserSource = new ParserSourceGLL<Token>(Token.EOF, tokenToNumber, genLiteral, numToString, tokenData, isLiteral, isTerminal, getLiteralNames, table, rules, rulesStart, fg.rules.leftSideArr, fg.startRule, fg.indexator.literalsEnd, fg.indexator.literalsStart, fg.indexator.termsEnd, fg.indexator.termsStart, fg.indexator.termCount, fg.indexator.nonTermCount, fg.indexator.literalsCount, fg.indexator.eofIndex, fg.rules.rulesCount, fg.indexator.fullCount, acceptEmptyInput, numIsTerminal, numIsNonTerminal, numIsLiteral, fg.canInferEpsilon, slots |> dict)
        parserSource
        //let gfg = GrammarFlowGraph()

        //type HGrammar<'TInt> =
        //    HGrammar of 'TInt * list<'TInt * list<'TInt>>

    let parse grammar graph = buildAbstractAst<Token> grammar graph

    let unpackExtension(ext: int64<extension>) =
        let ``2^32`` = 256L * 256L * 256L * 256L
        (int <| ext / ``2^32``, int <| ext % (``2^32`` * 1L<extension>))
    
    type ExtensionEdge = int * int64<extension>
    type ExtensionGraph =
        {
            start: ExtensionEdge
            terminals: list<int64<extension>>
            ands: Map<ExtensionEdge, ExtensionEdge * ExtensionEdge>
            ors: Map<ExtensionEdge, list<ExtensionEdge>>
        }
    
    let internal genDummy = -1, -1L<extension>
    let internal genTermEdge (i: int64<extension>) = 0, i
    let mutable private j = 0
    let internal genNodeEdge (i: int64<extension>) =<commit_msg>reduce code clean old comments<commit_after>//    F# .  http://fsharp.org
//   .   "  F#".

namespace YC.BioGraph

type Nuc = | A | C | G | U

type Edge = int * int * Nuc * bool

type Graph = {
    countOfVertex: int;
    edges: Edge[];
}

module Parser =
    open AbstractAnalysis.Common
    open Yard.Frontends.YardFrontend.Main
    open Yard.Generators.GLL.AbstractParser
    open AbstractAnalysis.Common
    open Yard.Generators.GLL
    open Yard.Generators.Common.FinalGrammar
    open Yard.Generators.Common.InitialConvert
    open Yard.Generators.Common.ASTGLL
    open Yard.Generators.GLL.ParserCommon
    open System.Collections.Generic
    open Yard.Generators.GLL
    open Yard.Generators.Common

    type Token =
        | A
        | C
        | G
        | U
        | EOF

    let graphParse graph'text =
        let gd = DotParser.parse graph'text
        let n = gd.Nodes.Count
        let g = new ParserInputGraph<Token>([|0..n|], [|n|])

        for i in 0..n do
            ignore <| g.AddVertex i

        for i in 0..n - 1 do
            ignore <| g.AddEdge(new ParserEdge<Token>(i, n, EOF))
    
        for edge in gd.Edges do
            let (x, y) = edge.Key
            let (a, b) = ref 0, ref 0
            if System.Int32.TryParse(x, a) && System.Int32.TryParse(y, b) then
                match edge.Value.Head.["label"] with
                | "A" -> ignore <| g.AddEdge(new ParserEdge<Token>(!a, !b, A))
                | "C" -> ignore <| g.AddEdge(new ParserEdge<Token>(!a, !b, C))
                | "G" -> ignore <| g.AddEdge(new ParserEdge<Token>(!a, !b, G))
                | "U" -> ignore <| g.AddEdge(new ParserEdge<Token>(!a, !b, U))
                | _ -> ()
        g
        (*
        let genGraphNWithEof n edges =
            let g = new ParserInputGraph<Token>([|0..n|], [|n|])

            for i in 0..n do
                ignore <| g.AddVertex i
    
            for i in 0..n - 1 do
                ignore <| g.AddEdge(new ParserEdge<Token>(i, n, EOF))
    
            for s, t, tok in edges do
                ignore <| g.AddEdge(new ParserEdge<Token>(s, t, tok))
    
            g

        let graph: ParserInputGraph<Token> =
            genGraphNWithEof 3 [0, 1, U; 1, 2, C]
        *)
    
    let mutable indToString = fun i -> ""
    let mutable tokenToNumber = fun t -> 0
    let tokenData (t:Token): obj = null

    let grmParse parser_text = 
        let text = parser_text
        (*let text = @"
        [<Start>]
        s: a b | b c | d
        a: A
        b: C
        c: G
        d: U"*)
        let grm = ParseText text "file.yrd"
        let icg = initialConvert grm
        let fg = FinalGrammar(icg.grammar.[0].rules, true)

        tokenToNumber <- function
            | A -> fg.indexator.termToIndex "A"
            | C -> fg.indexator.termToIndex "C"
            | G -> fg.indexator.termToIndex "G"
            | U -> fg.indexator.termToIndex "U"
            | EOF -> fg.indexator.eofIndex

        let genLiteral (s:string) (i:int): Token option = None
    
        let isLiteral (t:Token): bool = false
        let isTerminal (t:Token): bool = true
        let getLiteralNames = []

        let td = (Table fg).result
        let table = new System.Collections.Generic.Dictionary<int, int[]>()
        for k in td.Keys do
            table.Add(k, td.[k].ToArray())

        let rulesArr = Array.zeroCreate fg.rules.rulesCount
        for i = 0 to fg.rules.rulesCount-1 do
            rulesArr.[i] <- fg.rules.rightSide i

        let totalRulesLength = rulesArr |> Array.sumBy (fun x -> x.Length)
        let rules = Array.zeroCreate totalRulesLength
        let rulesStart = Array.zeroCreate <| fg.rules.rulesCount + 1
        let mutable cur = 0
        for i = 0 to fg.rules.rulesCount-1 do
            rulesStart.[i] <- cur
            for j = 0 to rulesArr.[i].Length-1 do
                rules.[cur] <- rulesArr.[i].[j]
                cur <- cur + 1
        rulesStart.[fg.rules.rulesCount] <- cur

        let acceptEmptyInput = true
        let numIsTerminal (i:int): bool = fg.indexator.termsStart <= i && i <= fg.indexator.termsEnd
        let numIsNonTerminal (i:int): bool = fg.indexator.isNonTerm i
        let numIsLiteral (i:int): bool = fg.indexator.literalsStart <= i && i <= fg.indexator.literalsEnd

        let numToString (n:int):string =
            if numIsTerminal n then
                fg.indexator.indexToTerm n
            elif numIsNonTerminal n then
                fg.indexator.indexToNonTerm n
            elif numIsLiteral n then
                fg.indexator.indexToLiteral n
            else string n
    
        indToString <- numToString

        let inline packRulePosition rule position = (int rule <<< 16) ||| int position

        let slots = new List<_>()
        slots.Add(packRulePosition -1 -1, 0)
        for i = 0 to fg.rules.rulesCount - 1 do
            let currentRightSide = fg.rules.rightSide i
            for j = 0 to currentRightSide.Length - 1 do
                if fg.indexator.isNonTerm currentRightSide.[j] then
                    let key = packRulePosition i (j + 1)
                    slots.Add(key, slots.Count)

        let parserSource = new ParserSourceGLL<Token>(Token.EOF, tokenToNumber, genLiteral, numToString, tokenData, isLiteral, isTerminal, getLiteralNames, table, rules, rulesStart, fg.rules.leftSideArr, fg.startRule, fg.indexator.literalsEnd, fg.indexator.literalsStart, fg.indexator.termsEnd, fg.indexator.termsStart, fg.indexator.termCount, fg.indexator.nonTermCount, fg.indexator.literalsCount, fg.indexator.eofIndex, fg.rules.rulesCount, fg.indexator.fullCount, acceptEmptyInput, numIsTerminal, numIsNonTerminal, numIsLiteral, fg.canInferEpsilon, slots |> dict)
        parserSource

    let parse grammar graph = buildAbstractAst<Token> grammar graph

    let unpackExtension(ext: int64<extension>) =
        let ``2^32`` = 256L * 256L * 256L * 256L
        (int <| ext / ``2^32``, int <| ext % (``2^32`` * 1L<extension>))
    
    type ExtensionEdge = int * int64<extension>
    type ExtensionGraph =
        {
            start: ExtensionEdge
            terminals: list<int64<extension>>
            ands: Map<ExtensionEdge, ExtensionEdge * ExtensionEdge>
            ors: Map<ExtensionEdge, list<ExtensionEdge>>
        }
    
    let internal genDummy = -1, -1L<extension>
    let internal genTermEdge (i: int64<extension>) = 0, i
    let mutable private j = 0
    let internal genNodeEdge (i: int64<extension>) =
{'fraction_non_alphanumeric': 0.11289582377237264, 'fraction_numerical': 0.011626128193360868, 'mean_word_length': 1.9289874551971327, 'pattern_counts': {'":': 0, '<': 110, '<?xml version=': 0, '>': 88, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3711923', 'n_tokens_mistral': 4368, 'n_tokens_neox': 4125, 'n_words': 1104}