<commit_before>{-# LANGUAGE FlexibleContexts  #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE RankNTypes        #-}

module Main where

import           Prelude                 hiding ((++))

import           Control.Concurrent.MVar
import           Control.Lens            hiding ((.=))
import           Control.Monad           (void)
import           Control.Monad.State     (evalStateT)
import           Control.Monad.Trans     (liftIO)
import           Data.Aeson              hiding (Success)
import           Data.Monoid
import qualified Data.Set                as Set
import qualified Data.Text.Encoding      as T
import qualified Misc
import           Network.Wai             (defaultRequest, rawPathInfo)
import           Test.Hspec
import           Web.Fn
import           Web.Larceny

import           Web.Offset

import           Common

runTests :: IO ()
runTests = hspec $ do
  Misc.tests
  larcenyFillTests
  cacheTests
  queryTests
  liveTests

main :: IO ()
main = runTests

--larcenyFillTests :: SpecM () ()
larcenyFillTests :: Spec
larcenyFillTests = do
  describe "<wpPosts>" $ do
    it "should show the title, id, and excerpt" $ do
      "<wpPosts><wpTitle/></wpPosts>" `shouldRender` "<i>Foo</i> bar"
      "<wpPosts><wpId/></wpPosts>" `shouldRender` "1"
      "<wpPosts><wpExcerpt/></wpPosts>" `shouldRender` "summary"
    it "should allow formating of dates" $
      "<wpPosts> \
      \  <wpDate> \
      \    <wpMonth format=\"%B\"/> <wpDay format=\"%-d\"/>, <wpYear /> \
      \  </wpDate> \
      \</wpPosts>" `shouldRender` "October 20, 2014"
    it "should render customly parsed fields" $
      "<wpPosts><wpDepartments><name /></wpDepartments></wpPosts>"
      `shouldRender` "some department"
  describe "<wpPage>" $
    it "should show the content" $
      "<wpPage name=a-first-page />" `shouldRender` "<b>rendered</b> page content"
  describe "<wpNoPostDuplicates/>" $ do
    it "should not duplicate any posts after call to wpNoPostDuplicates" $
      "<wpNoPostDuplicates/><wpPosts><wpTitle/></wpPosts><wpPosts><wpTitle/></wpPosts>" `shouldRender` "<i>Foo</i> bar"
    it "should ignore duplicates if they were rendered before wpNoPostDuplicates" $ do
      "<wpPosts><wpTitle/></wpPosts><wpNoPostDuplicates/><wpPosts><wpTitle/></wpPosts>" `shouldRender` "<i>Foo</i> bar<i>Foo</i> bar"
      "<wpPosts><wpTitle/></wpPosts><wpNoPostDuplicates/><wpPosts><wpTitle/></wpPosts><wpPosts><wpTitle/></wpPosts>" `shouldRender` "<i>Foo</i> bar<i>Foo</i> bar"
    it "should have no effect if it's at the end of the template" $
      "<wpPosts><wpTitle/></wpPosts><wpPosts><wpTitle/></wpPosts><wpNoPostDuplicates/>" `shouldRender` "<i>Foo</i> bar<i>Foo</i> bar"

  describe "<wpPostByPermalink>" $ do
    it "should query at a certain url" $ do
      record <- liftIO $ newMVar []
      let requestWithUrl = defaultRequest {rawPathInfo = T.encodeUtf8 "/2009/10/the-post/" }
      ctxt <- liftIO $ initializer
                        (Right $ Requester $ fauxRequester (Just record))
                        (CacheSeconds 10)
                        ""
      let ctxt' = setRequest ctxt
            $ (\(_,y) -> (requestWithUrl, y)) defaultFnRequest
      let s = _wpsubs ctxt'
      let tpl = toTpl "<wp><wpPostByPermalink><wpTitle/></wpPostByPermalink></wp"
      void $ evalStateT (runTemplate tpl [] s mempty) ctxt'
      liftIO (tryTakeMVar record) `shouldReturn` Just ["/wp/v2/posts?slug=the-post"]
    it "should render stuff" $ do
      ctxt <- initFauxRequestNoCache
      let requestWithUrl = defaultRequest {rawPathInfo = T.encodeUtf8 "/2009/10/the-post/"}
      let ctxt' = setRequest ctxt
                 $ (\(_,y) -> (requestWithUrl, y)) defaultFnRequest
      let s = view wpsubs ctxt'
      let tpl = toTpl "<wp><wpNoPostDuplicates/><wpPostByPermalink><wpTitle/></wpPostByPermalink><wpPosts limit=1><wpTitle/></wpPosts></wp>"
      rendered <- evalStateT (runTemplate tpl [] s mempty) ctxt'
      rendered `shouldBe` "<i>Foo</i> bar"

  describe "<wpCustom>" $
    it "should render an HTML comment if JSON field is null" $
      "<wpCustom endpoint=\"dev/null\"><wpThisIsNull /></wpCustom>" `shouldRender` "<!-- JSON field found, but value is null. -->"
  describe "<wpCustomDate>" $ do
    it "should parse a date field with the format string it's given" $
      "<wpCustomDate date=\"2013-04-26 10:11:52\" wp_format=\"%Y-%m-%d %H:%M:%S\"> \
      \   <wpDay />~<wpMonth />~<wpYear /> \
      \ </wpCustomDate>" `shouldRender` "26~04~2013"
    it "should format a date field with the format strings it's given" $
      "<wpCustomDate date=\"2013-04-26 10:11:52\" wp_format=\"%Y-%m-%d %H:%M:%S\"> \
      \   <wpMonth format=\"%B\"/> <wpDay format=\"%-d\"/>, <wpYear /> \
      \ </wpCustomDate>" `shouldRender` "April 26, 2013"
    it "should use default WordPress date format if none specified" $
      "<wpCustomDate date=\"2013-04-26 10:11:52\"> \
      \    <wpDay />~<wpMonth />~<wpYear /> \
      \ </wpCustomDate>" `shouldRender` "26~04~2013"
    it "should allow formatting the whole date in a single tag" $
      "<wpCustomDate date=\"2013-04-26 10:11:52\"> \
      \    <wpFullDate /> \
      \ </wpCustomDate>" `shouldRender` "04/26/13"

-- Caching tests

cacheTests :: Spec
cacheTests = do
  describe "should grab post from cache if it's there" $
      it "should render the post even w/o json source" $ do
        let (Object a2) = article2
        ctxt <- liftIO initNoRequestWithCache
        wpCacheSet' (view wordpress ctxt) (PostByPermalinkKey "2001" "10" "the-post")
                                          (enc [a2])
        ("single", ctxt) `shouldRenderAtUrlContaining` ("/2001/10/the-post/", "The post")

  describe "caching" $ do
    it "should find nothing for a non-existent post" $ do
      ctxt <- initNoRequestWithCache
      p <- wpCacheGet' (view wordpress ctxt) (PostByPermalinkKey "2000" "1" "the-article")
      p `shouldBe` Nothing
    it "should find something if there is a post in cache" $ do
      ctxt <- initNoRequestWithCache
      void $ wpCacheSet' (view wordpress ctxt) (PostByPermalinkKey "2000" "1" "the-article")
                                          (enc article1)
      p <- wpCacheGet' (view wordpress ctxt) (PostByPermalinkKey "2000" "1" "the-article")
      p `shouldBe` (Just $ enc article1)
    it "should not find single post after expire handler is called" $ do
         ctxt <- initNoRequestWithCache
         void $ wpCacheSet' (view wordpress ctxt) (PostByPermalinkKey "2000" "1" "the-article")
                                                  (enc article1)
         void $ wpExpirePost' (view wordpress ctxt) (PostByPermalinkKey "2000" "1" "the-article")
         wpCacheGet' (view wordpress ctxt) (PostByPermalinkKey "2000" "1" "the-article")
           >>= shouldBe Nothing
    it "should find post aggregates in cache" $
      do ctxt <- initNoRequestWithCache
         let key = PostsKey (Set.fromList [NumFilter 20, OffsetFilter 0])
         void $ wpCacheSet' (view wordpress ctxt) key ("[" <> enc article1 <> "]")
         void $ wpCacheGet' (view wordpress ctxt) key
           >>= shouldBe (Just $ "[" <> enc article1 <> "]")
    it "should not find post aggregates after expire handler is called" $
      do ctxt <- initNoRequestWithCache
         let key = PostsKey (Set.fromList [NumFilter 20, OffsetFilter 0])
         void $ wpCacheSet' (view wordpress ctxt) key ("[" <> enc article1 <> "]")
         void $ wpExpirePost' (view wordpress ctxt) (PostByPermalinkKey "2000" "1" "the-article")
         wpCacheGet' (view wordpress ctxt) key
           >>= shouldBe Nothing
    it "should find single post after expiring aggregates" $
      do ctxt <- initNoRequestWithCache
         void $ wpCacheSet' (view wordpress ctxt) (PostByPermalinkKey "2000" "1" "the-article")
                          (enc article1)
         void $ wpExpireAggregates' (view wordpress ctxt)
         wpCacheGet' (view wordpress ctxt) (PostByPermalinkKey "2000" "1" "the-article")
           >>= shouldNotBe Nothing
    it "should find a different single post after expiring another" $
      do ctxt <- initNoRequestWithCache
         let key1 = PostByPermalinkKey "2000" "1" "the-article"
             key2 = PostByPermalinkKey "2001" "2" "another-article"
         void $ wpCacheSet' (view wordpress ctxt) key1 (enc article1)
         void $ wpCacheSet' (view wordpress ctxt) key2 (enc article2)
         void $ wpExpirePost' (view wordpress ctxt) (PostByPermalinkKey "2000" "1" "the-article")
         wpCacheGet' (view wordpress ctxt) key2 >>= shouldBe (Just (enc article2))
    it "should be able to cache and retrieve post" $
      do ctxt <- initNoRequestWithCache
         let key = PostKey 200
         wpCacheSet' (view wordpress ctxt) key (enc article1)
         wpCacheGet' (view wordpress ctxt) key >>= shouldBe (Just (enc article1))

    describe "wpCustom" $ do
      it "should find single endpoint json after expire aggregates is called" $ do
        ctxt <- initNoRequestWithCache
        void $ wpCacheSet' (view wordpress ctxt) (EndpointKey "some-endpoint")
                                                  (enc article1)
        void $ wpExpireAggregates' (view wordpress ctxt)
        p <- wpCacheGet' (view wordpress ctxt) (EndpointKey "some-endpoint")
        p `shouldBe` (Just $ enc article1)

queryTests :: Spec
queryTests =
  describe "generate queries from <wpPosts>" $ do
      "<wpPosts></wpPosts>" `shouldQueryTo`
        ["/wp/v2/posts?offset=0&per_page=20"]
      "<wpPosts limit=2></wpPosts>" `shouldQueryTo`
        ["/wp/v2/posts?offset=0&per_page=20"]
      "<wpPosts offset=1 limit=1></wpPosts>" `shouldQueryTo`
        ["/wp/v2/posts?offset=1&per_page=20"]
      "<wpPosts offset=0 limit=1></wpPosts>" `shouldQueryTo`
        ["/wp/v2/posts?offset=0&per_page=20"]
      "<wpPosts limit=10 page=1></wpPosts>" `shouldQueryTo`
        ["/wp/v2/posts?offset=0&per_page=20"]
      "<wpPosts limit=10 page=2></wpPosts>" `shouldQueryTo`
        ["/wp/v2/posts?offset=20&per_page=20"]
      "<wpPosts num=2></wpPosts>" `shouldQueryTo`
        ["/wp/v2/posts?offset=0&per_page=2"]
      "<wpPosts num=2 page=2 limit=1></wpPosts>" `shouldQueryTo`
        ["/wp/v2/posts?offset=2&per_page=2"]
      "<wpPosts num=1 page=3></wpPosts>" `shouldQueryTo`
        ["/wp/v2/posts?offset=2&per_page=1"]<commit_msg>Remove test that was incorrect<commit_after>{-# LANGUAGE FlexibleContexts  #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE RankNTypes        #-}

module Main where

import           Prelude                 hiding ((++))

import           Control.Concurrent.MVar
import           Control.Lens            hiding ((.=))
import           Control.Monad           (void)
import           Control.Monad.State     (evalStateT)
import           Control.Monad.Trans     (liftIO)
import           Data.Aeson              hiding (Success)
import           Data.Monoid
import qualified Data.Set                as Set
import qualified Data.Text.Encoding      as T
import qualified Misc
import           Network.Wai             (defaultRequest, rawPathInfo)
import           Test.Hspec
import           Web.Fn
import           Web.Larceny

import           Web.Offset

import           Common

runTests :: IO ()
runTests = hspec $ do
  Misc.tests
  larcenyFillTests
  cacheTests
  queryTests
  liveTests

main :: IO ()
main = runTests

--larcenyFillTests :: SpecM () ()
larcenyFillTests :: Spec
larcenyFillTests = do
  describe "<wpPosts>" $ do
    it "should show the title, id, and excerpt" $ do
      "<wpPosts><wpTitle/></wpPosts>" `shouldRender` "<i>Foo</i> bar"
      "<wpPosts><wpId/></wpPosts>" `shouldRender` "1"
      "<wpPosts><wpExcerpt/></wpPosts>" `shouldRender` "summary"
    it "should allow formating of dates" $
      "<wpPosts> \
      \  <wpDate> \
      \    <wpMonth format=\"%B\"/> <wpDay format=\"%-d\"/>, <wpYear /> \
      \  </wpDate> \
      \</wpPosts>" `shouldRender` "October 20, 2014"
    it "should render customly parsed fields" $
      "<wpPosts><wpDepartments><name /></wpDepartments></wpPosts>"
      `shouldRender` "some department"
  describe "<wpPage>" $
    it "should show the content" $
      "<wpPage name=a-first-page />" `shouldRender` "<b>rendered</b> page content"
  describe "<wpNoPostDuplicates/>" $ do
    it "should not duplicate any posts after call to wpNoPostDuplicates" $
      "<wpNoPostDuplicates/><wpPosts><wpTitle/></wpPosts><wpPosts><wpTitle/></wpPosts>" `shouldRender` "<i>Foo</i> bar"
    it "should ignore duplicates if they were rendered before wpNoPostDuplicates" $ do
      "<wpPosts><wpTitle/></wpPosts><wpNoPostDuplicates/><wpPosts><wpTitle/></wpPosts>" `shouldRender` "<i>Foo</i> bar<i>Foo</i> bar"
      "<wpPosts><wpTitle/></wpPosts><wpNoPostDuplicates/><wpPosts><wpTitle/></wpPosts><wpPosts><wpTitle/></wpPosts>" `shouldRender` "<i>Foo</i> bar<i>Foo</i> bar"
    it "should have no effect if it's at the end of the template" $
      "<wpPosts><wpTitle/></wpPosts><wpPosts><wpTitle/></wpPosts><wpNoPostDuplicates/>" `shouldRender` "<i>Foo</i> bar<i>Foo</i> bar"

  describe "<wpPostByPermalink>" $ do
    it "should query at a certain url" $ do
      record <- liftIO $ newMVar []
      let requestWithUrl = defaultRequest {rawPathInfo = T.encodeUtf8 "/2009/10/the-post/" }
      ctxt <- liftIO $ initializer
                        (Right $ Requester $ fauxRequester (Just record))
                        (CacheSeconds 10)
                        ""
      let ctxt' = setRequest ctxt
            $ (\(_,y) -> (requestWithUrl, y)) defaultFnRequest
      let s = _wpsubs ctxt'
      let tpl = toTpl "<wp><wpPostByPermalink><wpTitle/></wpPostByPermalink></wp"
      void $ evalStateT (runTemplate tpl [] s mempty) ctxt'
      liftIO (tryTakeMVar record) `shouldReturn` Just ["/wp/v2/posts?slug=the-post"]
    it "should render stuff" $ do
      ctxt <- initFauxRequestNoCache
      let requestWithUrl = defaultRequest {rawPathInfo = T.encodeUtf8 "/2009/10/the-post/"}
      let ctxt' = setRequest ctxt
                 $ (\(_,y) -> (requestWithUrl, y)) defaultFnRequest
      let s = view wpsubs ctxt'
      let tpl = toTpl "<wp><wpNoPostDuplicates/><wpPostByPermalink><wpTitle/></wpPostByPermalink><wpPosts limit=1><wpTitle/></wpPosts></wp>"
      rendered <- evalStateT (runTemplate tpl [] s mempty) ctxt'
      rendered `shouldBe` "<i>Foo</i> bar"

  describe "<wpCustom>" $
    it "should render an HTML comment if JSON field is null" $
      "<wpCustom endpoint=\"dev/null\"><wpThisIsNull /></wpCustom>" `shouldRender` "<!-- JSON field found, but value is null. -->"
  describe "<wpCustomDate>" $ do
    it "should parse a date field with the format string it's given" $
      "<wpCustomDate date=\"2013-04-26 10:11:52\" wp_format=\"%Y-%m-%d %H:%M:%S\"> \
      \   <wpDay />~<wpMonth />~<wpYear /> \
      \ </wpCustomDate>" `shouldRender` "26~04~2013"
    it "should format a date field with the format strings it's given" $
      "<wpCustomDate date=\"2013-04-26 10:11:52\" wp_format=\"%Y-%m-%d %H:%M:%S\"> \
      \   <wpMonth format=\"%B\"/> <wpDay format=\"%-d\"/>, <wpYear /> \
      \ </wpCustomDate>" `shouldRender` "April 26, 2013"
    it "should use default WordPress date format if none specified" $
      "<wpCustomDate date=\"2013-04-26 10:11:52\"> \
      \    <wpDay />~<wpMonth />~<wpYear /> \
      \ </wpCustomDate>" `shouldRender` "26~04~2013"
    it "should allow formatting the whole date in a single tag" $
      "<wpCustomDate date=\"2013-04-26 10:11:52\"> \
      \    <wpFullDate /> \
      \ </wpCustomDate>" `shouldRender` "04/26/13"

-- Caching tests

cacheTests :: Spec
cacheTests = do
  describe "should grab post from cache if it's there" $
      it "should render the post even w/o json source" $ do
        let (Object a2) = article2
        ctxt <- liftIO initNoRequestWithCache
        wpCacheSet' (view wordpress ctxt) (PostByPermalinkKey "2001" "10" "the-post")
                                          (enc [a2])
        ("single", ctxt) `shouldRenderAtUrlContaining` ("/2001/10/the-post/", "The post")

  describe "caching" $ do
    it "should find nothing for a non-existent post" $ do
      ctxt <- initNoRequestWithCache
      p <- wpCacheGet' (view wordpress ctxt) (PostByPermalinkKey "2000" "1" "the-article")
      p `shouldBe` Nothing
    it "should find something if there is a post in cache" $ do
      ctxt <- initNoRequestWithCache
      void $ wpCacheSet' (view wordpress ctxt) (PostByPermalinkKey "2000" "1" "the-article")
                                          (enc article1)
      p <- wpCacheGet' (view wordpress ctxt) (PostByPermalinkKey "2000" "1" "the-article")
      p `shouldBe` (Just $ enc article1)
    it "should not find single post after expire handler is called" $ do
         ctxt <- initNoRequestWithCache
         void $ wpCacheSet' (view wordpress ctxt) (PostByPermalinkKey "2000" "1" "the-article")
                                                  (enc article1)
         void $ wpExpirePost' (view wordpress ctxt) (PostByPermalinkKey "2000" "1" "the-article")
         wpCacheGet' (view wordpress ctxt) (PostByPermalinkKey "2000" "1" "the-article")
           >>= shouldBe Nothing
    it "should find post aggregates in cache" $
      do ctxt <- initNoRequestWithCache
         let key = PostsKey (Set.fromList [NumFilter 20, OffsetFilter 0])
         void $ wpCacheSet' (view wordpress ctxt) key ("[" <> enc article1 <> "]")
         void $ wpCacheGet' (view wordpress ctxt) key
           >>= shouldBe (Just $ "[" <> enc article1 <> "]")
    it "should not find post aggregates after expire handler is called" $
      do ctxt <- initNoRequestWithCache
         let key = PostsKey (Set.fromList [NumFilter 20, OffsetFilter 0])
         void $ wpCacheSet' (view wordpress ctxt) key ("[" <> enc article1 <> "]")
         void $ wpExpirePost' (view wordpress ctxt) (PostByPermalinkKey "2000" "1" "the-article")
         wpCacheGet' (view wordpress ctxt) key
           >>= shouldBe Nothing
    it "should find single post after expiring aggregates" $
      do ctxt <- initNoRequestWithCache
         void $ wpCacheSet' (view wordpress ctxt) (PostByPermalinkKey "2000" "1" "the-article")
                          (enc article1)
         void $ wpExpireAggregates' (view wordpress ctxt)
         wpCacheGet' (view wordpress ctxt) (PostByPermalinkKey "2000" "1" "the-article")
           >>= shouldNotBe Nothing
    it "should find a different single post after expiring another" $
      do ctxt <- initNoRequestWithCache
         let key1 = PostByPermalinkKey "2000" "1" "the-article"
             key2 = PostByPermalinkKey "2001" "2" "another-article"
         void $ wpCacheSet' (view wordpress ctxt) key1 (enc article1)
         void $ wpCacheSet' (view wordpress ctxt) key2 (enc article2)
         void $ wpExpirePost' (view wordpress ctxt) (PostByPermalinkKey "2000" "1" "the-article")
         wpCacheGet' (view wordpress ctxt) key2 >>= shouldBe (Just (enc article2))
    it "should be able to cache and retrieve post" $
      do ctxt <- initNoRequestWithCache
         let key = PostKey 200
         wpCacheSet' (view wordpress ctxt) key (enc article1)
         wpCacheGet' (view wordpress ctxt) key >>= shouldBe (Just (enc article1))

queryTests :: Spec
queryTests =
  describe "generate queries from <wpPosts>" $ do
      "<wpPosts></wpPosts>" `shouldQueryTo`
        ["/wp/v2/posts?offset=0&per_page=20"]
      "<wpPosts limit=2></wpPosts>" `shouldQueryTo`
        ["/wp/v2/posts?offset=0&per_page=20"]
      "<wpPosts offset=1 limit=1></wpPosts>" `shouldQueryTo`
        ["/wp/v2/posts?offset=1&per_page=20"]
      "<wpPosts offset=0 limit=1></wpPosts>" `shouldQueryTo`
        ["/wp/v2/posts?offset=0&per_page=20"]
      "<wpPosts limit=10 page=1></wpPosts>" `shouldQueryTo`
        ["/wp/v2/posts?offset=0&per_page=20"]
      "<wpPosts limit=10 page=2></wpPosts>" `shouldQueryTo`
        ["/wp/v2/posts?offset=20&per_page=20"]
      "<wpPosts num=2></wpPosts>" `shouldQueryTo`
        ["/wp/v2/posts?offset=0&per_page=2"]
      "<wpPosts num=2 page=2 limit=1></wpPosts>" `shouldQueryTo`
        ["/wp/v2/posts?offset=2&per_page=2"]
      "<wpPosts num=1 page=3></wpPosts>" `shouldQueryTo`
        ["/wp/v2/posts?offset=2&per_page=1"]
{'fraction_non_alphanumeric': 0.13842198932173225, 'fraction_numerical': 0.025904686573067034, 'mean_word_length': 2.6981718464351006, 'pattern_counts': {'":': 0, '<': 315, '<?xml version=': 0, '>': 309, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2167379', 'n_tokens_mistral': 6879, 'n_tokens_neox': 6450, 'n_words': 1712}