<commit_before>{-# LANGUAGE BangPatterns          #-}
{-# LANGUAGE CPP                   #-}
{-# LANGUAGE DataKinds             #-}
{-# LANGUAGE ScopedTypeVariables   #-}
{-# LANGUAGE TypeOperators         #-}
{-# LANGUAGE TupleSections         #-}
{-# LANGUAGE TypeFamilies          #-}

import           Control.Monad ( foldM )
import           Control.Monad.Random ( MonadRandom, getRandomR )

#if __GLASGOW_HASKELL__ < 800
import           Data.List ( unfoldr )
#else
import           Data.List ( cycle, unfoldr )
#endif
import           Data.Semigroup ( (<>) )

import qualified Numeric.LinearAlgebra.Static as SA

import           Options.Applicative

import           Grenade
import           Grenade.Recurrent

-- The defininition for our simple recurrent network.
-- This file just trains a network to generate a repeating sequence
-- of 0 0 1.
--
-- The F and R types are Tagging types to ensure that the runner and
-- creation function know how to treat the layers.
type R = Recurrent

type RecNet = RecurrentNetwork '[ R (LSTM 1 4), R (LSTM 4 1)]
                               '[ 'D1 1, 'D1 4, 'D1 1 ]

type RecInput = RecurrentInputs '[ R (LSTM 1 4), R (LSTM 4 1)]

randomNet :: MonadRandom m => m RecNet
randomNet = randomRecurrent

netTest :: MonadRandom m => RecNet -> RecInput -> LearningParameters -> Int -> m (RecNet, RecInput)
netTest net0 i0 rate iterations =
    foldM trainIteration (net0,i0) [1..iterations]
  where
    trainingCycle = cycle [c 0, c 0, c 1]

    trainIteration (net, io) _ = do
      dropping <- getRandomR (0, 2)
      count    <- getRandomR (5, 30)
      let t     = drop dropping trainingCycle
      let example = ((,Nothing) <$> take count t) ++ [(t !! count, Just $ t !! (count + 1))]<commit_msg>Simplify recurrent example<commit_after>{-# LANGUAGE BangPatterns          #-}
{-# LANGUAGE CPP                   #-}
{-# LANGUAGE DataKinds             #-}
{-# LANGUAGE ScopedTypeVariables   #-}
{-# LANGUAGE TypeOperators         #-}
{-# LANGUAGE TupleSections         #-}
{-# LANGUAGE TypeFamilies          #-}

import           Control.Monad ( foldM )
import           Control.Monad.Random ( MonadRandom, getRandomR )

#if __GLASGOW_HASKELL__ < 800
import           Data.List ( unfoldr )
#else
import           Data.List ( cycle, unfoldr )
#endif
import           Data.Semigroup ( (<>) )

import qualified Numeric.LinearAlgebra.Static as SA

import           Options.Applicative

import           Grenade
import           Grenade.Recurrent

-- The defininition for our simple recurrent network.
-- This file just trains a network to generate a repeating sequence
-- of 0 0 1.
--
-- The F and R types are Tagging types to ensure that the runner and
-- creation function know how to treat the layers.
type R = Recurrent

type RecNet = RecurrentNetwork '[ R (LSTM 1 1)]
                               '[ 'D1 1, 'D1 1]

type RecInput = RecurrentInputs '[ R (LSTM 1 1)]

randomNet :: MonadRandom m => m RecNet
randomNet = randomRecurrent

netTest :: MonadRandom m => RecNet -> RecInput -> LearningParameters -> Int -> m (RecNet, RecInput)
netTest net0 i0 rate iterations =
    foldM trainIteration (net0,i0) [1..iterations]
  where
    trainingCycle = cycle [c 0, c 0, c 1]

    trainIteration (net, io) _ = do
      dropping <- getRandomR (0, 2)
      count    <- getRandomR (5, 30)
      let t     = drop dropping trainingCycle
      let example = ((,Nothing) <$> take count t) ++ [(t !! count, Just $ t !! (count + 1))]
{'fraction_non_alphanumeric': 0.10684377707190297, 'fraction_numerical': 0.017903551833670227, 'mean_word_length': 2.318007662835249, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5869705', 'n_tokens_mistral': 1151, 'n_tokens_neox': 1078, 'n_words': 363}