<commit_before>﻿module Process

open Nessos.Argu
open Newtonsoft.Json
open System.IO
open System.Diagnostics
open System.Text
open Type
open Culture

let convertLineToUTF8 (line: string) =
    // Create two different encodings.
    let codepage = get_codepage
    let systemCodePage = Encoding.GetEncoding(codepage)
    let utf8 = Encoding.UTF8;
    let utf8Bytes = Encoding.Convert(systemCodePage, utf8, systemCodePage.GetBytes(line.ToCharArray()))
    let utf8Str = utf8.GetString(utf8Bytes)
    utf8Str

let pretty_print (json: string) =
    try
        let parsedJson = JsonConvert.DeserializeObject(json)
        JsonConvert.SerializeObject(parsedJson, Formatting.Indented)
    with
        | :? JsonReaderException -> sprintf "%s" json // When error occurred, printing json string as-is.

let check_groonga (config: Config) =
    System.IO.File.Exists <| config.Path

let check_dbpath (config: Config) =
    System.IO.File.Exists <| config.DBPath

let start_groonga (config: Config) (inputs: string list) =
    let encoding = config.DBEncoding |> Encoding.GetEncoding<commit_msg>Apply newline style to formatted json string<commit_after>﻿module Process

open Nessos.Argu
open Newtonsoft.Json
open System.IO
open System.Diagnostics
open System.Text
open Type
open Culture

let convertLineToUTF8 (line: string) =
    // Create two different encodings.
    let codepage = get_codepage
    let systemCodePage = Encoding.GetEncoding(codepage)
    let utf8 = Encoding.UTF8;
    let utf8Bytes = Encoding.Convert(systemCodePage, utf8, systemCodePage.GetBytes(line.ToCharArray()))
    let utf8Str = utf8.GetString(utf8Bytes)
    utf8Str

let pretty_print (json: string) =
    try
        let parsedJson = JsonConvert.DeserializeObject(json)
        JsonConvert.SerializeObject(parsedJson, Formatting.Indented) |> sprintf "%s\n"
    with
        | :? JsonReaderException -> sprintf "%s" json // When error occurred, printing json string as-is.

let check_groonga (config: Config) =
    System.IO.File.Exists <| config.Path

let check_dbpath (config: Config) =
    System.IO.File.Exists <| config.DBPath

let start_groonga (config: Config) (inputs: string list) =
    let encoding = config.DBEncoding |> Encoding.GetEncoding
{'fraction_non_alphanumeric': 0.09103199639477241, 'fraction_numerical': 0.00811176205497972, 'mean_word_length': 4.011286681715576, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3788146', 'n_tokens_mistral': 677, 'n_tokens_neox': 692, 'n_words': 181}