<commit_before>{-# LANGUAGE OverloadedStrings #-}
module ParserSpec where

import Control.Applicative
import Test.Hspec
import Data.Monoid ((<>))
import qualified Data.Aeson as AE
import Data.Aeson (Value(..))
import qualified Data.ByteString.Char8 as BS
import qualified Data.ByteString.Lazy as BL
import qualified Data.Text as T
import Control.Monad (forM_)
import Data.Text.Encoding (encodeUtf8)
import qualified Data.Vector as Vec
import qualified Data.HashMap.Strict as HMap
import System.Directory (getDirectoryContents)
import Data.Int
import Data.Word

import Data.JsonStream.Parser

-- During the tests the single quotes are replaced with double quotes in the strings as
-- otherwise it would be unreadable in haskell

todquotes :: BS.ByteString -> BS.ByteString
todquotes = BS.map squotes
  where
    squotes '\'' = '"'
    squotes x = x

parse :: Parser a -> BS.ByteString -> [a]
parse parser text = parseByteString parser (todquotes text)

testRemaining :: Parser a -> BS.ByteString -> BS.ByteString
testRemaining parser startdata = loop (runParser' parser startdata)
  where
    loop (ParseNeedData _) = error "Not enough data."
    loop (ParseDone rem1) = rem1
    loop (ParseFailed err) = error err
    loop (ParseYield _ np) = loop np


specBase :: Spec
specBase = describe "Basic parsing" $ do
  it "Parses null values" $ do
    let test = "[null,3]"
        res = parse (arrayOf value) test :: [Maybe Int]
    res `shouldBe` [Nothing, Just 3 ]

  it "Parses bool values" $ do
    let test = "[true,false]"
        res = parse (arrayOf value) test :: [Bool]
    res `shouldBe` [True, False]

  it "Parses string values with special chracters" $ do
    let test = "['" `BS.append` (encodeUtf8 "žluť")  `BS.append` "', '\\n\\b\\r\\'', '\\u0041\\u0078\\u0161']"
        res = parse (arrayOf value) test :: [T.Text]
    res `shouldBe` ["\382lu\357","\n\b\r\"","Ax\353"]

  it "Parses fractional values with exponent" $ do
    let test = "[1, 2.5, -3.6, 6e1, -3.2e-2]"
        res = parse (arrayOf value) test :: [Double]
    show res `shouldBe` "[1.0,2.5,-3.6,60.0,-3.2e-2]"

  it "Parses objects 1" $ do
    let test = "{'key1': 'value2', 'key2': 'value2'}"
        res = parse (objectItems value) test :: [(T.Text, AE.Value)]
    res `shouldBe` [("key1",String "value2"),("key2",String "value2")]

specObjComb :: Spec
specObjComb = describe "Object accesors" $ do
  it "objectWithKey works" $ do
    let test = "[{'name': 'John', 'age': 20}, {'age': 30, 'name': 'Frank' } ]"
        msg = parse (arrayOf $ (,) <$> objectWithKey "name" value <*> objectWithKey "age" value) test :: [(T.Text,Int)]
    msg `shouldBe` [("John",20),("Frank",30)]

  it "yield test 1" $ do
    let test = "[{'key1': [1,2,3], 'key2': [5,6,7]}]"
        msg1 = parse (arrayOf $ objectItems value) test :: [(T.Text, [Int])]
        msg2 = parse (arrayOf $ objectItems $ arrayOf value) test :: [(T.Text, Int)]
    msg1 `shouldBe` [("key1",[1,2,3]),("key2",[5,6,7])]
    msg2 `shouldBe` [("key1",1),("key1",2),("key1",3),("key2",5),("key2",6),("key2",7)]

  it "<*> test 1 reverse keys" $ do
    let test = "[{'key1': [1,2], 'key2': [5,6], 'key3': [8,9]}]"
        parser = arrayOf $ (,) <$> objectWithKey "key2" (arrayOf value) <*> objectWithKey "key1" (arrayOf value)
        msg = parse parser test :: [(Int, Int)]
    msg `shouldBe` [(5,1),(5,2),(6,1),(6,2)]

  it "<> test 1" $ do
    let test = "[{'key1': [1,2], 'key2': [5,6], 'key3': [8,9]}]"
        parser = arrayOf $ objectWithKey "key1" (arrayOf value) <> objectWithKey "key2" (arrayOf value)
        msg = parse parser test :: [Int]
    msg `shouldBe` [1,2,5,6]

  it "<|> returns first items even if second is in previous chunk" $ do
    let test = ["{\"error\":1, ", "\"values\":[2,3,4]}"]
        parser =  "values" .: arrayOf integer
                  <|> "error" .: integer
        msg = parseLazyByteString parser (BL.fromChunks test) :: [Int]
    msg `shouldBe` [2,3,4]
  it "<|> returns second item if first does not match" $ do
    let test = ["{\"error\":1, ", "\"values\":[true,null,false]}"]
        parser =  "values" .: arrayOf integer
                  <|> ("error" .: integer)
        msg = parseLazyByteString parser (BL.fromChunks test) :: [Int]
    msg `shouldBe` [1]

  it "arrayFound generates events" $ do
    let test = ["[[1,2,3],true,[],false,{\"key\":1}]"]
        parser = arrayOf (arrayFound 10 20 (1 .! integer))
        msg = parseLazyByteString parser (BL.fromChunks test) :: [Int]
    msg `shouldBe` [10,2,20,10,20]

  it "objectFound generates events" $ do
    let test = ["[[1,2,3],true,[],false,{\"key\":1}]"]
        parser = arrayOf (objectFound 10 20 ("key" .: integer))
        msg = parseLazyByteString parser (BL.fromChunks test) :: [Int]
    msg `shouldBe` [10,1,20]

  it "Has working byteString parser" $ do
    let test = ["[\"abcd\\n\\rxyz\"]"]
        parser = arrayOf byteString :: Parser BS.ByteString
        msg = parseLazyByteString parser (BL.fromChunks test) :: [BS.ByteString]
    msg `shouldBe` ["abcd\\n\\rxyz"]

specEdge :: Spec
specEdge = describe "Edge cases" $ do
  it "Correct incremental parsing 1" $ do
    let msg1 = "[ {\"test1\"  :[1,true,false,null,-3.591e+1,[12,13]], \"test2\":\"123\\r\\n\\\"\\u0041\"}]"
        pmsg = BL.fromChunks $ map BS.singleton msg1
        res = parseLazyByteString value pmsg :: [AE.Value]
    res `shouldBe` [Array (Vec.fromList [Object $ HMap.fromList [("test2",String "123\r\n\"A"),("test1",Array (Vec.fromList [Number 1.0,Bool True,Bool False,Null,Number (-35.91),Array (Vec.fromList [Number 12.0,Number 13.0])]))]])]

  it "Correct incremental parsing 2" $ do
    let msg1 = "{\"test1\"  :[1,true,false,null,-3.591e+1,[12,13]], \"test2\":\"test2string\"}"
        pmsg = BL.fromChunks $ map BS.singleton msg1
        res = parseLazyByteString ("test2" .: string) pmsg :: [T.Text]
    res `shouldBe` ["test2string"]
<commit_msg>Fix test suite to work with aeson-2.0<commit_after>{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE CPP                 #-}
module ParserSpec where

import Control.Applicative
import Test.Hspec
import Data.Monoid ((<>))
import qualified Data.Aeson as AE
import Data.Aeson (Value(..))
import qualified Data.ByteString.Char8 as BS
import qualified Data.ByteString.Lazy as BL
import qualified Data.Text as T
import Control.Monad (forM_)
import Data.Text.Encoding (encodeUtf8)
import qualified Data.Vector as Vec
import System.Directory (getDirectoryContents)
import Data.Int
import Data.Word
#if MIN_VERSION_aeson(2,0,0)
import qualified Data.Aeson.KeyMap           as AEK
#else
import qualified Data.HashMap.Strict as HMap
#endif

import Data.JsonStream.Parser

-- During the tests the single quotes are replaced with double quotes in the strings as
-- otherwise it would be unreadable in haskell

todquotes :: BS.ByteString -> BS.ByteString
todquotes = BS.map squotes
  where
    squotes '\'' = '"'
    squotes x = x

parse :: Parser a -> BS.ByteString -> [a]
parse parser text = parseByteString parser (todquotes text)

testRemaining :: Parser a -> BS.ByteString -> BS.ByteString
testRemaining parser startdata = loop (runParser' parser startdata)
  where
    loop (ParseNeedData _) = error "Not enough data."
    loop (ParseDone rem1) = rem1
    loop (ParseFailed err) = error err
    loop (ParseYield _ np) = loop np


specBase :: Spec
specBase = describe "Basic parsing" $ do
  it "Parses null values" $ do
    let test = "[null,3]"
        res = parse (arrayOf value) test :: [Maybe Int]
    res `shouldBe` [Nothing, Just 3 ]

  it "Parses bool values" $ do
    let test = "[true,false]"
        res = parse (arrayOf value) test :: [Bool]
    res `shouldBe` [True, False]

  it "Parses string values with special chracters" $ do
    let test = "['" `BS.append` (encodeUtf8 "žluť")  `BS.append` "', '\\n\\b\\r\\'', '\\u0041\\u0078\\u0161']"
        res = parse (arrayOf value) test :: [T.Text]
    res `shouldBe` ["\382lu\357","\n\b\r\"","Ax\353"]

  it "Parses fractional values with exponent" $ do
    let test = "[1, 2.5, -3.6, 6e1, -3.2e-2]"
        res = parse (arrayOf value) test :: [Double]
    show res `shouldBe` "[1.0,2.5,-3.6,60.0,-3.2e-2]"

  it "Parses objects 1" $ do
    let test = "{'key1': 'value2', 'key2': 'value2'}"
        res = parse (objectItems value) test :: [(T.Text, AE.Value)]
    res `shouldBe` [("key1",String "value2"),("key2",String "value2")]

specObjComb :: Spec
specObjComb = describe "Object accesors" $ do
  it "objectWithKey works" $ do
    let test = "[{'name': 'John', 'age': 20}, {'age': 30, 'name': 'Frank' } ]"
        msg = parse (arrayOf $ (,) <$> objectWithKey "name" value <*> objectWithKey "age" value) test :: [(T.Text,Int)]
    msg `shouldBe` [("John",20),("Frank",30)]

  it "yield test 1" $ do
    let test = "[{'key1': [1,2,3], 'key2': [5,6,7]}]"
        msg1 = parse (arrayOf $ objectItems value) test :: [(T.Text, [Int])]
        msg2 = parse (arrayOf $ objectItems $ arrayOf value) test :: [(T.Text, Int)]
    msg1 `shouldBe` [("key1",[1,2,3]),("key2",[5,6,7])]
    msg2 `shouldBe` [("key1",1),("key1",2),("key1",3),("key2",5),("key2",6),("key2",7)]

  it "<*> test 1 reverse keys" $ do
    let test = "[{'key1': [1,2], 'key2': [5,6], 'key3': [8,9]}]"
        parser = arrayOf $ (,) <$> objectWithKey "key2" (arrayOf value) <*> objectWithKey "key1" (arrayOf value)
        msg = parse parser test :: [(Int, Int)]
    msg `shouldBe` [(5,1),(5,2),(6,1),(6,2)]

  it "<> test 1" $ do
    let test = "[{'key1': [1,2], 'key2': [5,6], 'key3': [8,9]}]"
        parser = arrayOf $ objectWithKey "key1" (arrayOf value) <> objectWithKey "key2" (arrayOf value)
        msg = parse parser test :: [Int]
    msg `shouldBe` [1,2,5,6]

  it "<|> returns first items even if second is in previous chunk" $ do
    let test = ["{\"error\":1, ", "\"values\":[2,3,4]}"]
        parser =  "values" .: arrayOf integer
                  <|> "error" .: integer
        msg = parseLazyByteString parser (BL.fromChunks test) :: [Int]
    msg `shouldBe` [2,3,4]
  it "<|> returns second item if first does not match" $ do
    let test = ["{\"error\":1, ", "\"values\":[true,null,false]}"]
        parser =  "values" .: arrayOf integer
                  <|> ("error" .: integer)
        msg = parseLazyByteString parser (BL.fromChunks test) :: [Int]
    msg `shouldBe` [1]

  it "arrayFound generates events" $ do
    let test = ["[[1,2,3],true,[],false,{\"key\":1}]"]
        parser = arrayOf (arrayFound 10 20 (1 .! integer))
        msg = parseLazyByteString parser (BL.fromChunks test) :: [Int]
    msg `shouldBe` [10,2,20,10,20]

  it "objectFound generates events" $ do
    let test = ["[[1,2,3],true,[],false,{\"key\":1}]"]
        parser = arrayOf (objectFound 10 20 ("key" .: integer))
        msg = parseLazyByteString parser (BL.fromChunks test) :: [Int]
    msg `shouldBe` [10,1,20]

  it "Has working byteString parser" $ do
    let test = ["[\"abcd\\n\\rxyz\"]"]
        parser = arrayOf byteString :: Parser BS.ByteString
        msg = parseLazyByteString parser (BL.fromChunks test) :: [BS.ByteString]
    msg `shouldBe` ["abcd\\n\\rxyz"]



specEdge :: Spec
specEdge = describe "Edge cases" $ do
  it "Correct incremental parsing 1" $ do
    let msg1 = "[ {\"test1\"  :[1,true,false,null,-3.591e+1,[12,13]], \"test2\":\"123\\r\\n\\\"\\u0041\"}]"
        pmsg = BL.fromChunks $ map BS.singleton msg1
        res = parseLazyByteString value pmsg :: [AE.Value]
#if MIN_VERSION_aeson(2,0,0)
    res `shouldBe` [Array (Vec.fromList [Object $ AEK.fromList [("test2",String "123\r\n\"A"),("test1",Array (Vec.fromList [Number 1.0,Bool True,Bool False,Null,Number (-35.91),Array (Vec.fromList [Number 12.0,Number 13.0])]))]])]
#else
    res `shouldBe` [Array (Vec.fromList [Object $ HMap.fromList [("test2",String "123\r\n\"A"),("test1",Array (Vec.fromList [Number 1.0,Bool True,Bool False,Null,Number (-35.91),Array (Vec.fromList [Number 12.0,Number 13.0])]))]])]
#endif

  it "Correct incremental parsing 2" $ do
    let msg1 = "{\"test1\"  :[1,true,false,null,-3.591e+1,[12,13]], \"test2\":\"test2string\"}"
        pmsg = BL.fromChunks $ map BS.singleton msg1
        res = parseLazyByteString ("test2" .: string) pmsg :: [T.Text]
    res `shouldBe` ["test2string"]

{'fraction_non_alphanumeric': 0.18288480472595997, 'fraction_numerical': 0.04028552674762061, 'mean_word_length': 3.323873714083008, 'pattern_counts': {'":': 16, '<': 27, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '307157', 'n_tokens_mistral': 4561, 'n_tokens_neox': 4306, 'n_words': 1217}