<commit_before>﻿module YC.Bio.RNA.SearchFSA

open QuickGraph
open Argu
open System
open YC.Bio.GraphLoader
open AbstractAnalysis.Common
open Yard.Generators.GLL.ParserCommon
open Yard.Generators.GLL.AbstractParser
open YC.Bio.RNA.Search.Structs
open YC.Bio.RNA.IO
open Microsoft.FSharp.Collections
open System.IO
open System.Collections.Generic
open System.Diagnostics
open InfernalApi

let filterRnaParsingResult (graph : EdgeCompressedGraphInput) (searchCfg : SearchConfig) res = 
    let getPaths s e (lengths : HashSet<_>) = 
        let maxLength = 
            lengths
            |> Seq.max
    
        let rec dfs start curLength = 
            let toReturn = new List<List<TaggedEdge<_,_>>>()
            for edge in graph.OutEdges start do
                let newLength = curLength + edge.Tag.length
                if edge.Target = e && newLength
                                   |> lengths.Contains
                then new List<_>([ edge ]) |> toReturn.Add
                if newLength < maxLength then 
                    let paths = dfs edge.Target newLength
                    paths |> ResizeArray.iter (fun (x : List<_>) -> x.Add edge)
                    toReturn.AddRange paths
            toReturn
        
        let paths = dfs s 0 |> ResizeArray.map ResizeArray.rev
        if lengths.Contains 0 then new List<_>() |> paths.Add
        paths
    
    let result = new ResizeArray<_>()

    res 
    |> Array.filter (fun (f,t,l) -> int l >= searchCfg.LowLengthLimit && int l <= searchCfg.HighLengthLimit)
    |> Array.groupBy (fun (f,t,l) -> 
            let start =
                match graph.GetEdgeFromPackedPod f with
                | Some v -> v.Target, Some v
                | None -> graph.VerticesBackMap.[int (graph.GetPosOnEdgeFromPackedPod f)], None
            let _end =
                match graph.GetEdgeFromPackedPod t with
                | Some v -> v.Source, Some v
                | None -> graph.VerticesBackMap.[int (graph.GetPosOnEdgeFromPackedPod t)], None
            start, _end)
    |> Array.iter (fun ((eFrom, eTo), a) ->
        let letgths =
            a |> Array.map (fun (f,t,l) -> 
                let prefixLength = 
                    match snd eFrom with
                    | Some v -> v.Tag.str.Length - (int <| graph.GetPosOnEdgeFromPackedPod f)
                    | _ -> 0
                let suffixLEngth = 
                    match snd eTo with
                    | Some v -> (int <| graph.GetPosOnEdgeFromPackedPod t)
                    | None -> 0
                int l - prefixLength - suffixLEngth)
            |> fun a -> new HashSet<_>(a)
        let midlePaths = getPaths (fst eFrom) (fst eTo) letgths
        
        midlePaths
        |> ResizeArray.map (fun a -> 
            let res = new ResizeArray<_>()
            match snd eFrom with
            | Some v -> res.Add v
            | None -> ()
            res.AddRange a
            match snd eTo with
            | Some v -> res.Add v
            | None -> ()
            res)
        |> fun r -> result.AddRange r
        )

    result

let pathToString (searchCfg : SearchConfig) (edges: ResizeArray<TaggedEdge<_,BioGraphEdgeLbl>>) =
    let data =
        edges
        |> ResizeArray.map (fun e -> e.Tag.str |> Array.map (fun i -> searchCfg.NumToString.[int i]) |> String.concat "")
        |> String.concat ""

    let metadata =
        edges
        |> ResizeArray.map(fun e -> e.Tag.id |> string )
        |> String.concat "; "
    "> " + metadata + "\n" + data + "\n"

let parsingResultsProcessor () =
        MailboxProcessor.Start(fun inbox -> 
            let rec loop n = 
                async { 
                    let! msg = inbox.Receive()
                    match msg with
                    | PData(graph, cfg, res) -> 
                        try 
                            let pathToPrint = filterRnaParsingResult graph cfg res
                            pathToPrint
                            |> ResizeArray.map (pathToString cfg)
                            |> fun strs -> System.IO.File.AppendAllLines(cfg.OutFileName, strs)
                        with e -> printfn "ERROR in parsing results postprocessing! %A" e.Message
                        return! loop n
                    | PDie ch -> ch.Reply()
                }
            loop 0)
    
let searchInBioGraphs (searchCfg : SearchConfig) (graphs : EdgeCompressedGraphInput[]) agentsCount = 
    let start = System.DateTime.Now
    let postprocessor = parsingResultsProcessor ()
    let agent name = 
        MailboxProcessor.Start(fun inbox -> 
            let rec loop n = 
                async { 
                    let! msg = inbox.Receive()<commit_msg>Improve edges to fa printing.<commit_after>﻿module YC.Bio.RNA.SearchFSA

open QuickGraph
open Argu
open System
open YC.Bio.GraphLoader
open AbstractAnalysis.Common
open Yard.Generators.GLL.ParserCommon
open Yard.Generators.GLL.AbstractParser
open YC.Bio.RNA.Search.Structs
open YC.Bio.RNA.IO
open Microsoft.FSharp.Collections
open System.IO
open System.Collections.Generic
open System.Diagnostics
open InfernalApi

let filterRnaParsingResult (graph : EdgeCompressedGraphInput) (searchCfg : SearchConfig) res = 
    let getPaths s e (lengths : HashSet<_>) = 
        let maxLength = 
            lengths
            |> Seq.max
    
        let rec dfs start curLength = 
            let toReturn = new List<List<TaggedEdge<_,_>>>()
            for edge in graph.OutEdges start do
                let newLength = curLength + edge.Tag.length
                if edge.Target = e && newLength
                                   |> lengths.Contains
                then new List<_>([ edge ]) |> toReturn.Add
                if newLength < maxLength then 
                    let paths = dfs edge.Target newLength
                    paths |> ResizeArray.iter (fun (x : List<_>) -> x.Add edge)
                    toReturn.AddRange paths
            toReturn
        
        let paths = dfs s 0 |> ResizeArray.map ResizeArray.rev
        if lengths.Contains 0 then new List<_>() |> paths.Add
        paths
    
    let result = new ResizeArray<_>()

    res 
    |> Array.filter (fun (f,t,l) -> int l >= searchCfg.LowLengthLimit && int l <= searchCfg.HighLengthLimit)
    |> Array.groupBy (fun (f,t,l) -> 
            let start =
                match graph.GetEdgeFromPackedPod f with
                | Some v -> v.Target, Some v
                | None -> graph.VerticesBackMap.[int (graph.GetPosOnEdgeFromPackedPod f)], None
            let _end =
                match graph.GetEdgeFromPackedPod t with
                | Some v -> v.Source, Some v
                | None -> graph.VerticesBackMap.[int (graph.GetPosOnEdgeFromPackedPod t)], None
            start, _end)
    |> Array.iter (fun ((eFrom, eTo), a) ->
        let letgths =
            a |> Array.map (fun (f,t,l) -> 
                let prefixLength = 
                    match snd eFrom with
                    | Some v -> v.Tag.str.Length - (int <| graph.GetPosOnEdgeFromPackedPod f)
                    | _ -> 0
                let suffixLEngth = 
                    match snd eTo with
                    | Some v -> (int <| graph.GetPosOnEdgeFromPackedPod t)
                    | None -> 0
                int l - prefixLength - suffixLEngth)
            |> fun a -> new HashSet<_>(a)
        let midlePaths = getPaths (fst eFrom) (fst eTo) letgths
        
        midlePaths
        |> ResizeArray.map (fun a -> 
            let res = new ResizeArray<_>()
            match snd eFrom with
            | Some v -> res.Add v
            | None -> ()
            res.AddRange a
            match snd eTo with
            | Some v -> res.Add v
            | None -> ()
            res)
        |> fun r -> result.AddRange r
        )

    result

let pathToString (searchCfg : SearchConfig) id (edges: ResizeArray<TaggedEdge<_,BioGraphEdgeLbl>>) =
    let data =
        edges
        |> ResizeArray.map (fun e -> e.Tag.str |> Array.map (fun i -> searchCfg.NumToString.[int i]) |> String.concat "")
        |> String.concat ""

    let metadata =
        edges
        |> ResizeArray.map(fun e -> e.Tag.id |> string )
        |> String.concat "; "
    "> " + id + " | " + metadata + "\n" + data + "\n"

let parsingResultsProcessor () =
    let edgesGlobalCounter = ref 0
    MailboxProcessor.Start(fun inbox -> 
        let rec loop n = 
            async { 
                let! msg = inbox.Receive()
                match msg with
                | PData(graph, cfg, res) -> 
                    try 
                        let pathToPrint = filterRnaParsingResult graph cfg res
                        pathToPrint
                        |> ResizeArray.map (fun e -> 
                            incr edgesGlobalCounter
                            pathToString cfg (string !edgesGlobalCounter) e)
                        |> fun strs -> System.IO.File.AppendAllLines(cfg.OutFileName, strs)
                    with e -> printfn "ERROR in parsing results postprocessing! %A" e.Message
                    return! loop n
                | PDie ch -> ch.Reply()
            }
        loop 0)
    
let searchInBioGraphs (searchCfg : SearchConfig) (graphs : EdgeCompressedGraphInput[]) agentsCount = 
    let start = System.DateTime.Now
    let postprocessor = parsingResultsProcessor ()
    let agent name = 
        MailboxProcessor.Start(fun inbox -> 
            let rec loop n = 
                async { 
                    let! msg = inbox.Receive()
{'fraction_non_alphanumeric': 0.09354635274332562, 'fraction_numerical': 0.001156190876602901, 'mean_word_length': 1.489534275248561, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 134, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '898403', 'n_tokens_mistral': 2769, 'n_tokens_neox': 2559, 'n_words': 730}