<commit_before>{-# LANGUAGE BangPatterns,OverloadedStrings #-}
module VarName (parseVarName, 
                nsTail,
                nsQualifiers,
                parseNSTag,
                parseProc,
                VarName(..), 
                arrName,
                isArr,
                showVN, 
                NSQual(..), 
                NSTag(..),
                asGlobal,
                isGlobalQual,
                noNsQual,
                splitWith,
                nsSep,
                varNameTests) where
import Util
import qualified Data.ByteString.Char8 as B
import Data.ByteString (findSubstrings)
import Test.HUnit


data NSQual a = NSQual !(Maybe NSTag) !a deriving (Eq,Show)

data NSTag = NS !Bool [BString] deriving (Eq,Show,Ord)

data VarName = VarName { vnName :: !BString, vnInd :: Maybe BString } deriving (Eq,Show)

class BStringable a where
  toBStr :: a -> BString

instance BStringable NSTag where
  toBStr (NS g lst) = B.append (if g then nsSep else B.empty) (B.intercalate nsSep lst)

arrName !an !ind = VarName an (Just ind)
{-# INLINE arrName #-}

isArr (VarName _ (Just _)) = True
isArr _                    = False

nsSep :: BString
nsSep = "::"

isGlobalQual (Just (NS v _)) = v
isGlobalQual _               = False
{-# INLINE isGlobalQual #-}

noNsQual nst = case nst of
         Nothing              -> True
         Just (NS False [])   -> True
         _                    -> False
{-# INLINE noNsQual #-}

asGlobal (Just (NS _ lst)) = Just (NS True lst)
asGlobal Nothing           = Just (NS True [])

parseNSQual ns = case parseNSTag ns of
                  NS False [s] -> NSQual Nothing s
                  NS gq []     -> NSQual (Just (NS gq [])) ""
                  NS gq nsl    -> NSQual (Just (NS gq (init nsl))) (last nsl)
{-# INLINE parseNSQual #-}

	       <commit_msg>expanded toBStr.. really should rename the class and it's singular function.<commit_after>{-# LANGUAGE BangPatterns,OverloadedStrings #-}
module VarName (parseVarName, 
                nsTail,
                nsQualifiers,
                parseNSTag,
                parseProc,
                VarName(..), 
                arrName,
                isArr,
                showVN, 
                NSQual(..), 
                NSTag(..),
                asGlobal,
                isGlobalQual,
                noNsQual,
                splitWith,
                nsSep,
                BStringable(..),
                varNameTests) where
import Util
import qualified Data.ByteString.Char8 as B
import Data.ByteString (findSubstrings)
import Test.HUnit


data NSQual a = NSQual !(Maybe NSTag) !a deriving (Eq,Show)

data NSTag = NS !Bool [BString] deriving (Eq,Show,Ord)

data VarName = VarName { vnName :: !BString, vnInd :: Maybe BString } deriving (Eq,Show)

class BStringable a where
  toBStr :: a -> BString

instance BStringable B.ByteString where
  toBStr = id
instance BStringable NSTag where
  toBStr (NS g lst) = B.append (if g then nsSep else B.empty) (B.intercalate nsSep lst)

instance (BStringable a) => BStringable (NSQual a) where
  toBStr (NSQual (Just t) s) = B.append (toBStr t) (toBStr s)
  toBStr (NSQual _ s) = toBStr s

arrName !an !ind = VarName an (Just ind)
{-# INLINE arrName #-}

isArr (VarName _ (Just _)) = True
isArr _                    = False

nsSep :: BString
nsSep = "::"

isGlobalQual (Just (NS v _)) = v
isGlobalQual _               = False
{-# INLINE isGlobalQual #-}

noNsQual nst = case nst of
         Nothing              -> True
         Just (NS False [])   -> True
         _                    -> False
{-# INLINE noNsQual #-}

asGlobal (Just (NS _ lst)) = Just (NS True lst)
asGlobal Nothing           = Just (NS True [])

parseNSQual ns = case parseNSTag ns of
                  NS False [s] -> NSQual Nothing s
                  NS gq []     -> NSQual (Just (NS gq [])) ""
                  NS gq nsl    -> NSQual (Just (NS gq (init nsl))) (last nsl)
{-# INLINE parseNSQual #-}

	       
{'fraction_non_alphanumeric': 0.10128495842781557, 'fraction_numerical': 0.0005039052658100278, 'mean_word_length': 1.6878808395396072, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4836348', 'n_tokens_mistral': 1289, 'n_tokens_neox': 1225, 'n_words': 381}