<commit_before>﻿module ParserTests

open FParsec
open FlexSearch.Core
open FlexSearch.Core.Parsers
open FsUnit
open Fuchu
open System
open System.Threading

let parser = new FlexParser()

let test p str = 
    match FParsec.CharParsers.run p str with
    | Success(result, _, _) -> Assert.AreEqual(1, 1)
    | Failure(errorMsg, _, _) -> Assert.AreEqual(1, 2, errorMsg)

let test2 str = 
    match parser.Parse(str) with
    | Choice1Of2(a) -> Assert.AreEqual(1, 1)
    | Choice2Of2(errorMsg) -> Assert.AreEqual(1, 2, errorMsg)

[<Tests>]
let parserTests = 
    testList "Parser Tests" 
        [ testCase "Single escape character should be accepted" 
          <| fun _ -> test FlexSearch.Core.Parsers.stringLiteral "'abc \\' pqr'"
          
          testList "List of Values test" 
              [ for exp in [ 1, "['abc']"
                             2, "['abc','pqr']"
                             3, "['abc'  ,  'pqr']"
                             4, "[         'abc'          ]"
                             5, "[    'abc'    ]" ] -> 
                    testCase (sprintf "%i: Function call should parse" (fst (exp))) 
                    <| fun _ -> test FlexSearch.Core.Parsers.listOfValues (snd (exp)) ]
          
          testList "Simple Expressions" 
              [ for exp in [ 1, "abc eq 'a'"
                             2, "not abc eq 'a'"
                             3, "(abc eq '1')"<commit_msg>ParserTests.fs update to handle choice<commit_after>﻿module ParserTests

open FParsec
open FlexSearch.Core
open FlexSearch.Core.Parsers
open FsUnit
open Fuchu
open System
open System.Threading

let parser = new FlexParser()

let test p str = 
    match FParsec.CharParsers.run p str with
    | Success(result, _, _) -> Assert.AreEqual(1, 1)
    | Failure(errorMsg, _, _) -> Assert.AreEqual(1, 2, errorMsg)

let test2 str = 
    match parser.Parse(str) with
    | Choice1Of2(a) -> Assert.AreEqual(1, 1)
    | Choice2Of2(errorMsg) -> Assert.AreEqual(1, 2, errorMsg.DeveloperMessage)

[<Tests>]
let parserTests = 
    testList "Parser Tests" 
        [ testCase "Single escape character should be accepted" 
          <| fun _ -> test FlexSearch.Core.Parsers.stringLiteral "'abc \\' pqr'"
          
          testList "List of Values test" 
              [ for exp in [ 1, "['abc']"
                             2, "['abc','pqr']"
                             3, "['abc'  ,  'pqr']"
                             4, "[         'abc'          ]"
                             5, "[    'abc'    ]" ] -> 
                    testCase (sprintf "%i: Function call should parse" (fst (exp))) 
                    <| fun _ -> test FlexSearch.Core.Parsers.listOfValues (snd (exp)) ]
          
          testList "Simple Expressions" 
              [ for exp in [ 1, "abc eq 'a'"
                             2, "not abc eq 'a'"
                             3, "(abc eq '1')"
{'fraction_non_alphanumeric': 0.12205270457697642, 'fraction_numerical': 0.015256588072122053, 'mean_word_length': 1.6108597285067874, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3500788', 'n_tokens_mistral': 915, 'n_tokens_neox': 860, 'n_words': 241}