<commit_before>module Main where
import Data.Either (either)
import Data.Serialize
import Data.ByteString.Base64
import qualified Data.ByteString.Char8 as B
import Text.Regex.Posix
import Control.Monad (replicateM)
import Data.Maybe (catMaybes)
import Control.Applicative ((<|>))
import qualified Data.Binary as DB
import qualified Data.ByteString.UTF8 as UTF8
import Data.Char (chr)

xs = ["&AOkA6QDp-", "&bElbVw-", "&bElbVw-/&byJbVw-"]


eitherIntVal :: B.ByteString -> Either String [Integer]
eitherIntVal = runGet (do 
    xs <- replicateM 5 (Just `fmap` getWord16be <|> return Nothing)
    return $ map fromIntegral $ catMaybes xs)



intVal :: B.ByteString -> [Integer]
intVal x = either error id (eitherIntVal x)

charNums :: String -> [Int]
charNums = map fromIntegral . intVal . decodeLenient . B.pack

-- divides utf7 by word dividers 
divide :: String -> [String]
divide x = map shave $ getAllTextMatches $ x =~ "&([^-]*)-" :: [String]
    where shave s = drop 1 $ take (length s - 1) s 

main = do
    mapM_ (putStrLn.show. divide) xs
    mapM_ (\x -> mapM_ putStrLn x ) $ map (map (map chr . charNums) . divide) xs

<commit_msg>use Parsec instead of Regex, much better code<commit_after>module Main where

import Text.ParserCombinators.Parsec hiding ((<|>))

import Data.Either (either)
import Data.Serialize
import Data.ByteString.Base64
import qualified Data.ByteString.Char8 as B
import Text.Regex.Posix
import Control.Monad (replicateM)
import Data.Maybe (catMaybes)
import Control.Applicative ((<|>))
import qualified Data.Binary as DB
import qualified Data.ByteString.UTF8 as UTF8
import Data.Char (chr)

xs = ["&AOkA6QDp-", "&bElbVw-", "&bElbVw-/&byJbVw-"]


eitherIntVal :: B.ByteString -> Either String [Integer]
eitherIntVal = runGet (do 
    xs <- replicateM 5 (Just `fmap` getWord16be <|> return Nothing)
    return $ map fromIntegral $ catMaybes xs)



intVal :: B.ByteString -> [Integer]
intVal x = either error id (eitherIntVal x)

charNums :: String -> [Int]
charNums = map fromIntegral . intVal . decodeLenient . B.pack


plain :: GenParser Char st String
plain = many1 (noneOf "&")

special :: GenParser Char st String
special = do
  char '&'
  x <- many (noneOf "-")
  char '-'
  return $ (map chr . charNums)  x

word' = (try plain) <|> special

parseUtf7Words :: GenParser Char st String
parseUtf7Words = fmap concat (many1 word')

parseUtf7 :: String -> Either ParseError String
parseUtf7 inp = parse parseUtf7Words "(unknown)" inp

main = do

    mapM_ (\x ->  do
        let r = either (error . show) id $ parseUtf7 x 
        putStrLn r
      ) xs


{'fraction_non_alphanumeric': 0.1202164669501353, 'fraction_numerical': 0.011209895632006184, 'mean_word_length': 3.774907749077491, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '563662', 'n_tokens_mistral': 942, 'n_tokens_neox': 888, 'n_words': 283}