<commit_before>module NGramCrackers.Ops.Text
(
  bigrams
, trigrams
, getNGramsFromText
, getNGramsFromTextList
, getAlphasOnlyToText
, normToList
, getWordFrequency
, countWords
, typeTokenRatio
, (<#>)
) where

import qualified Data.Text as T
import Data.Char (isAlpha, isAlphaNum, isSpace, toLower)
import Data.List (map, nub)

import NGramCrackers.Utilities.List

{-| Infix synonym for T.append. Handy for gluing bits of Text together -}
(<#>) :: T.Text -> T.Text -> T.Text
(<#>) = T.append

{-| Extract bigrams from a string -}
bigrams :: T.Text -> [T.Text]
bigrams = getNGramsFromText 2 

{-| Extract trigrams from a string -}
trigrams :: T.Text -> [T.Text]
trigrams = getNGramsFromText 3 

{-| Extract n-grams from a string -}
getNGramsFromText :: Int -> T.Text -> [T.Text]
getNGramsFromText n packed -- packed is a packed T.Text string
    | n < 0     = error "n must be a positive integer less than 7"
    | n > 7     = error "n must be a positive integer less than 7"
    | otherwise = map T.unwords $ getNGramsFromTextList n wordList
                    where wordList = T.words packed

getNGramsFromTextList :: Int -> [T.Text] -> [[T.Text]]
getNGramsFromTextList = getNSeqFromList

{-| Return only alphabetic characters from a string and return the
    result as a string. Output of this function may need processing
    into a list, tuple, etc. -}
getAlphasOnlyToText :: T.Text -> T.Text
getAlphasOnlyToText = T.filter (\char -> isAlpha char || isSpace char)

{-| Return only alphanumeric characters from a string and return the
    result as a List.-}
normToList :: T.Text -> [T.Text]
normToList = T.words . getAlphasOnlyToText .  T.toLower 

{-| Get frequency of a single word's occurance in a string. Is eta-reduction
    the easiest reading way to do this function? The arguments are fairly
    instructive. However, the type declaration does say what kind of args
    it takes.  With type synonyms or further exploring the type system,
    the declaration would be more informative-}

getWordFrequency:: T.Text -> T.Text -> Int<commit_msg>Code prettying<commit_after>module NGramCrackers.Ops.Text
(
  bigrams
, trigrams
, getNGramsFromText
, getNGramsFromTextList
, getAlphasOnlyToText
, normToList
, getWordFrequency
, countWords
, typeTokenRatio
, (<#>)
) where

import qualified Data.Text as T
import Data.Char (isAlpha, isAlphaNum, isSpace, toLower)
import Data.List (map, nub)

import NGramCrackers.Utilities.List

{-| Infix synonym for T.append. Handy for gluing bits of Text together -}
(<#>) :: T.Text -> T.Text -> T.Text
(<#>) = T.append

{-| Extract bigrams from a string -}
bigrams :: T.Text -> [T.Text]
bigrams = getNGramsFromText 2 

{-| Extract trigrams from a string -}
trigrams :: T.Text -> [T.Text]
trigrams = getNGramsFromText 3 

{-| Extract n-grams from a string -}
getNGramsFromText :: Int -> T.Text -> [T.Text]
getNGramsFromText n packed -- packed is a packed T.Text string
    | n < 0     = error "n must be a positive integer less than 7"
    | n > 7     = error "n must be a positive integer less than 7"
    | otherwise = map T.unwords $ getNGramsFromTextList n wordList
                    where wordList = T.words packed

getNGramsFromTextList :: Int -> [T.Text] -> [[T.Text]]
getNGramsFromTextList = getNSeqFromList

{-| Return only alphabetic characters from a string and return the
    result as a string. Output of this function may need processing
    into a list, tuple, etc. -}
getAlphasOnlyToText :: T.Text -> T.Text
getAlphasOnlyToText = T.filter (\char -> isAlpha char || isSpace char)

{-| Return only alphanumeric characters from a string and return the
    result as a List.-}
normToList :: T.Text -> [T.Text]
normToList = T.words . getAlphasOnlyToText . T.toLower

{-| Get frequency of a single word's occurance in a string. Is eta-reduction
    the easiest reading way to do this function? The arguments are fairly
    instructive. However, the type declaration does say what kind of args
    it takes.  With type synonyms or further exploring the type system,
    the declaration would be more informative-}

getWordFrequency:: T.Text -> T.Text -> Int
{'fraction_non_alphanumeric': 0.09919766593727207, 'fraction_numerical': 0.0029175784099197666, 'mean_word_length': 3.462039045553145, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1980759', 'n_tokens_mistral': 1364, 'n_tokens_neox': 1301, 'n_words': 474}