<commit_before>﻿module MyStem.Sharp.Wrapper

open System.Diagnostics
open System.Text
open System.IO
open Newtonsoft.Json.Linq

type Defention(gr:string,lex:string) =
    member this.Gr = gr
    member this.Lex = lex

type WordDefention(text, analysis:Defention[]) = 
    member this.Text = text
    member this.Analysis = if isNull analysis then Array.empty else analysis
    member this.GetText() : string = 
        match this.Analysis |> Array.length with
        | 0 -> this.Text
        | _ -> analysis.[0].Lex

type Lemmatizer (path: string) = 
    
    let lockObject = ref 4

    let createStartInfo path =
        let startInfo = new ProcessStartInfo()
        do
            startInfo.UseShellExecute <- false
            startInfo.RedirectStandardOutput <- true
            startInfo.RedirectStandardError <- true
            startInfo.RedirectStandardInput <- true
            startInfo.CreateNoWindow <- true
            startInfo.WindowStyle <- ProcessWindowStyle.Hidden
            startInfo.FileName <- path
            startInfo.Arguments <- " -lcdi --format json"
        startInfo
    
    let createProcess path:Process =
        let proc = new Process (StartInfo = createStartInfo path)
        proc.Start()
        proc
    
    let mystemProc = createProcess path
    let reader = new StreamReader(mystemProc.StandardOutput.BaseStream, Encoding.UTF8)

    let getProcessOutput text (proc:Process) (reader:StreamReader) = 
        let buffer = Encoding.UTF8.GetBytes(text + "\r\n")
        proc.StandardInput.BaseStream.Write(buffer,0,buffer.Length)
        proc.StandardInput.BaseStream.Flush()
        reader.ReadLine()
                             
    let call (action: ('T -> 'R)) (obj:'T) : 'R =
        action obj

    member this.Lemmatize (text:string) : WordDefention[] =
        lock lockObject (fun () -> getProcessOutput text mystemProc reader)
            |> JArray.Parse
            |> call (fun a -> a.ToObject<list<WordDefention>>())
            |> List.toArray
<commit_msg>clean text before lemmatize for avoid suspension<commit_after>﻿module MyStem.Sharp.Wrapper

open System.Diagnostics
open System.Text
open System.IO
open Newtonsoft.Json.Linq
open System.Text.RegularExpressions

type Defention(gr:string,lex:string) =
    member this.Gr = gr
    member this.Lex = lex

type WordDefention(text, analysis:Defention[]) = 
    member this.Text = text
    member this.Analysis = if isNull analysis then Array.empty else analysis
    member this.GetText() : string = 
        match this.Analysis |> Array.length with
        | 0 -> this.Text
        | _ -> analysis.[0].Lex

type Lemmatizer (path: string) = 
    
    let lockObject = ref 4
    let cleanRegex = new Regex(@"[^\w\d-]", RegexOptions.Compiled ||| RegexOptions.IgnoreCase);

    let createStartInfo path =
        let startInfo = new ProcessStartInfo()
        do
            startInfo.UseShellExecute <- false
            startInfo.RedirectStandardOutput <- true
            startInfo.RedirectStandardError <- true
            startInfo.RedirectStandardInput <- true
            startInfo.CreateNoWindow <- true
            startInfo.WindowStyle <- ProcessWindowStyle.Hidden
            startInfo.FileName <- path
            startInfo.Arguments <- " -lcdi --format json"
        startInfo
    
    let createProcess path:Process =
        let proc = new Process (StartInfo = createStartInfo path)
        proc.Start()
        proc
    
    let mystemProc = createProcess path
    let reader = new StreamReader(mystemProc.StandardOutput.BaseStream, Encoding.UTF8)

    let getProcessOutput (proc:Process) (reader:StreamReader) text = 
        let buffer = Encoding.UTF8.GetBytes(text + "\r\n")
        proc.StandardInput.BaseStream.Write(buffer,0,buffer.Length)
        proc.StandardInput.BaseStream.Flush()
        reader.ReadLine()
                             
    let call (action: ('T -> 'R)) (obj:'T) : 'R =
        action obj

    member this.Lemmatize (text:string) : WordDefention[] =
        cleanRegex.Replace(text, " ")
            |> lock lockObject (fun () -> getProcessOutput mystemProc reader)
            |> JArray.Parse
            |> call (fun a -> a.ToObject<list<WordDefention>>())
            |> List.toArray

{'fraction_non_alphanumeric': 0.09704840613931523, 'fraction_numerical': 0.002833530106257379, 'mean_word_length': 2.372611464968153, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3282716', 'n_tokens_mistral': 1199, 'n_tokens_neox': 1149, 'n_words': 288}