<commit_before>{-# LANGUAGE ConstraintKinds, FlexibleContexts, FlexibleInstances, ImplicitParams, LambdaCase, MultiParamTypeClasses, OverloadedStrings, PackageImports, TemplateHaskell #-}

module Formura.MPICxx.Translate where

import           Control.Applicative
import           Control.Concurrent(threadDelay)
import qualified Control.Exception as X
import           Control.Lens
import           Control.Monad
import "mtl"     Control.Monad.RWS
import           Data.Char (toUpper, isAlphaNum)
import           Data.Foldable (toList)
import           Data.List (zip4,zip5, zip6)
import qualified Data.Map as M
import           Data.Maybe
import qualified Data.Set as S
import qualified Data.Text as T
import qualified Data.Text.Lens as T
import qualified Data.Text.IO as T
import qualified Data.Yaml as Y
import           System.Directory
import           System.FilePath.Lens
import           System.Process
import           Text.Trifecta (failed, raiseErr)


import qualified Formura.Annotation as A
import           Formura.Annotation.Boundary
import           Formura.Annotation.Representation
import           Formura.Compiler
import           Formura.CommandLineOption
import           Formura.Geometry
import           Formura.GlobalEnvironment
import           Formura.Language.Combinator (subFix)
import           Formura.NumericalConfig
import           Formura.OrthotopeMachine.Graph
import           Formura.Syntax
import           Formura.Vec
import           Formura.MPICxx.Cut hiding (cut)

showC :: Show a => a -> T.Text
showC = T.pack . show

parens :: T.Text -> T.Text
parens x = "(" <> x <> ")"

brackets :: T.Text -> T.Text
brackets x = "[" <> x <> "]"

braces :: T.Text -> T.Text
braces x = "{" <> x <> "}"

newtype VariableName = VariableName T.Text


-- | The struct for generating unique names, and holds already given names.
data NamingState = NamingState
  { _alreadyGivenNames :: S.Set T.Text
  , _alreadyGivenLocalNames :: S.Set T.Text
  , _alreadyDeclaredResourceNames :: S.Set T.Text
  , _freeNameCounter :: Integer
  , _freeLocalNameCounter :: Integer
  , _nodeIDtoLocalName :: M.Map MMNodeID T.Text
  , _loopIndexNames :: Vec T.Text
  , _loopIndexOffset :: Vec Int
  , _loopExtentNames :: Vec T.Text
  }
makeClassy ''NamingState

defaultNamingState = NamingState
  { _alreadyGivenNames = S.empty
  , _alreadyGivenLocalNames = S.empty
  , _alreadyDeclaredResourceNames = S.empty
  , _freeNameCounter = 0
  , _freeLocalNameCounter = 0
  , _nodeIDtoLocalName = M.empty
  , _loopIndexNames = PureVec ""
  , _loopIndexOffset = 0
  , _loopExtentNames = PureVec ""
  }

type MPIPlanSelector = Bool

data TranState = TranState
  { _tranSyntacticState :: CompilerSyntacticState
  , _tsNumericalConfig :: NumericalConfig
  , _tsNamingState :: NamingState
  , _theProgram :: Program
  , _theMMProgram :: MMProgram
  , _theGraph :: MMGraph
  , _tsMPIPlanSelection :: MPIPlanSelector
  , _tsMPIPlanMap :: M.Map MPIPlanSelector MPIPlan
  , _tsCommonStaticBox :: Box
  , _tsCxxTemplateWithMacro :: T.Text
  }
makeClassy ''TranState


instance HasCompilerSyntacticState TranState where
  compilerSyntacticState = tranSyntacticState
instance HasNumericalConfig TranState where
  numericalConfig = tsNumericalConfig
instance HasMachineProgram TranState MMInstruction OMNodeType where
  machineProgram = theMMProgram
instance HasNamingState TranState where
  namingState = tsNamingState
instance HasMPIPlan TranState where
  mPIPlan =
    let
      gettr s = fromJust $ M.lookup (s^.tsMPIPlanSelection) (s^.tsMPIPlanMap)
      settr s a = s & tsMPIPlanMap %~ M.insert (s^.tsMPIPlanSelection) a
    in lens gettr settr

data CProgram = CProgram { _headerFileContent :: T.Text, _sourceFileContent :: T.Text,
                         _auxFilesContent :: M.Map FilePath T.Text}
                deriving (Eq, Ord, Show)
makeLenses ''CProgram

tellH :: (MonadWriter CProgram m) => T.Text -> m ()
tellH txt = tell $ CProgram txt "" M.empty
tellC :: (MonadWriter CProgram m) => T.Text -> m ()
tellC txt = tell $ CProgram "" txt M.empty
tellBoth :: (MonadWriter CProgram m) => T.Text -> m ()
tellBoth txt = tell $ CProgram txt txt  M.empty
tellF :: (MonadWriter CProgram m) => FilePath -> T.Text -> m ()
tellF fn txt = tell $ CProgram "" ""  (M.singleton fn txt)

tellHLn :: (MonadWriter CProgram m) => T.Text -> m ()
tellHLn txt = tellH $ txt <> "\n"
tellCLn :: (MonadWriter CProgram m) => T.Text -> m ()
tellCLn txt = tellC $ txt <> "\n"
tellBothLn :: (MonadWriter CProgram m) => T.Text -> m ()
tellBothLn txt = tellBoth $ txt <> "\n"
tellFLn :: (MonadWriter CProgram m) => FilePath -> T.Text -> m ()
tellFLn fn txt = tellF fn $ txt <> "\n"


instance Monoid CProgram where
  mempty = CProgram "" "" M.empty
  mappend (CProgram h1 c1 f1) (CProgram h2 c2 f2) = CProgram (h1 <> h2) (c1 <> c2) (M.unionWith (<>) f1 f2)


type TranM = CompilerMonad GlobalEnvironment CProgram TranState

-- * Parallel code generation

-- | generate new free global name based on given identifier,
--   and prevent further generation of that name
genFreeName :: IdentName -> TranM T.Text
genFreeName = genFreeName' True

-- | generate new free local name based on given identifier,
--   and prevent further generation of that name within current scope
genFreeLocalName :: IdentName -> TranM T.Text
genFreeLocalName = genFreeName' False

-- | base function for giving names
genFreeName' :: Bool -> IdentName -> TranM T.Text
genFreeName' isGlobal ident = do
  aggNames <- use alreadyGivenNames
  aglNames <- use alreadyGivenLocalNames
  let initName = T.pack ident
      agNames = aggNames <> aglNames
      nCounter :: Lens' TranState Integer
      nCounter = if isGlobal then freeNameCounter else freeLocalNameCounter
      go = do
        ctr <- use nCounter
        let tmpName = initName <> "_" <> showC ctr
        if S.member tmpName agNames
          then (nCounter += 1) >> go
          else return tmpName
  givenName <- if S.member initName agNames then go else return initName
  (if isGlobal then alreadyGivenNames else alreadyGivenLocalNames) %= S.insert givenName
  return givenName

-- | read all numerical config from the Formura source program
setNumericalConfig :: WithCommandLineOption => TranM ()
setNumericalConfig = do
  dim <- view dimension
  ivars <- view axesNames

  mnc <- liftIO $ Y.decodeFile ncFilePath
  nc <- case mnc of
     Nothing -> raiseErr $ failed $ "cannot parse numerical config .yaml file: " ++ show ncFilePath
     Just x -> return x

  tsNumericalConfig .= nc


  when (length (nc ^. ncMPIGridShape) /= dim) $
    raiseErr $ failed $ "mpi_grid_shape need exactly " ++ show dim ++ " elements."
  when (length (nc ^. ncIntraNodeShape) /= dim) $
    raiseErr $ failed $ "intra_node_shape need exactly " ++ show dim ++ " elements."
  return ()

-- | prepare unique name for everyone
setNamingState :: TranM ()
setNamingState = do
  stateVars <- use omStateSignature
  alreadyGivenNames .= (S.fromList $ map T.pack $ M.keys stateVars)

  ans <- view axesNames
  lins <- traverse (genFreeName . ("i"++)) ans
  loopIndexNames .= lins

  luns <- traverse (genFreeName . ("N"++) . map toUpper) ans
  loopExtentNames .= luns

  let nameNode :: MMNode -> TranM MMNode
      nameNode nd = do
        let initName = case A.viewMaybe nd  of
                        Just (SourceName n) -> n
                        _                   -> "g"
        cName <- genFreeName initName
        return $ nd & A.annotation %~ A.set (VariableName cName)

  gr <- use omInitGraph
  gr2 <- flip traverse gr $ nameNode
  omInitGraph .= gr2

  gr <- use omStepGraph
  gr2 <- flip traverse gr $ nameNode
  omStepGraph .= gr2


-- | Generate C type declaration for given language.
genTypeDecl :: IdentName -> TypeExpr -> TranM T.Text
genTypeDecl name typ = case typ of
  ElemType "void" -> return ""
  ElemType "Rational" -> return $ "double " <> T.pack name
  ElemType x -> return $ T.pack  x <> " " <> T.pack name
  GridType _ x -> do
    body <- genTypeDecl name x
    if body == "" then return ""
      else do
        sz <- use ncIntraNodeShape
        let szpt = foldMap (brackets . showC) sz
        return $ body <> szpt
  _ -> raiseErr $ failed $ "Cannot translate type to C: " ++ show typ


elemTypeOfResource :: ResourceT a b -> TranM TypeExpr
elemTypeOfResource (ResourceStatic sname _) = do
  ssMap <- use omStateSignature
  let Just typ = M.lookup sname ssMap
  case typ of
    ElemType _ -> return typ
    GridType _ etyp -> return etyp
elemTypeOfResource (ResourceOMNode nid _) = do
  mmProg <- use omStepGraph
  let Just nd = M.lookup nid mmProg
  case nd ^.nodeType of
    ElemType x -> return $ ElemType x
    GridType _ etyp -> return $ subFix etyp

tellMPIRequestDecl :: T.Text -> TranM ()
tellMPIRequestDecl name = do
  adrn <- use alreadyDeclaredResourceNames
  case S.member name adrn of
    True -> return ()
    False -> do
      alreadyDeclaredResourceNames %= S.insert name
      tellH "extern "
      tellBothLn $ "MPI_Request "<>name<>";\n"
tellResourceDecl :: T.Text -> ResourceT a b -> Box -> TranM ()
tellResourceDecl = tellResourceDecl' False

tellResourceDecl' :: Bool -> T.Text -> ResourceT a b -> Box -> TranM ()
tellResourceDecl' isInClass name rsc box0 = do
  adrn <- use alreadyDeclaredResourceNames
  case S.member name adrn || name == "" of
    True -> return ()
    False -> do
      alreadyDeclaredResourceNames %= S.insert name

      typ <- elemTypeOfResource rsc
      let szpt = foldMap (brackets . showC) sz
          sz = box0 ^.upperVertex - box0 ^. lowerVertex

      decl <- case typ of
        ElemType "void" -> return ""
        ElemType "Rational" -> return $ "double " <> name <> szpt
        ElemType x -> return $ T.pack  x <> " " <> name <> szpt
        _ -> raiseErr $ failed $ "Cannot translate type to C: " ++ show typ
      when (decl /= "") $ do

        when isInClass $ do
          tellH decl
          tellHLn ";"
        when (not isInClass) $ do
          tellH "extern "
          tellBoth decl
          tellBothLn ";"


--       case rsc of
--         ResourceStatic _ _ -> do
--             tellH "extern "
--             tellBoth decl
--             tellBothLn ";"
--         _ -> do
--           when (decl /= "") $ tellCLn $ decl <> ";"


tellFacetDecl :: FacetID -> [RidgeID] -> TranM ()
tellFacetDecl f rs = do
  let name = T.pack $ toCName f
  tellH $ "struct " <> name <> "{"

  forM_ rs $ \rk -> do
    name <- nameRidgeResource' True rk SendRecv
    tellResourceDecl' True name rsc box0

  tellH "};"

  tellH $ "extern struct " <> name <> " " <> name <> "_Send;"
  tellH $ "extern struct " <> name <> " " <> name <> "_Recv;"
  tellC $ "struct " <> name <> " " <> name <> "_Send;"
  tellC $ "struct " <> name <> " " <> name <> "_Recv;"
  return ()


toCName :: Show a => a -> IdentName
toCName a = postfix $ fix $ go False $ prefix $ show a
  where
    go _ [] = []<commit_msg>Move ridge decl within Facet<commit_after>{-# LANGUAGE ConstraintKinds, FlexibleContexts, FlexibleInstances, ImplicitParams, LambdaCase, MultiParamTypeClasses, OverloadedStrings, PackageImports, TemplateHaskell #-}

module Formura.MPICxx.Translate where

import           Control.Applicative
import           Control.Concurrent(threadDelay)
import qualified Control.Exception as X
import           Control.Lens
import           Control.Monad
import "mtl"     Control.Monad.RWS
import           Data.Char (toUpper, isAlphaNum)
import           Data.Foldable (toList)
import           Data.List (zip4,zip5, zip6)
import qualified Data.Map as M
import           Data.Maybe
import qualified Data.Set as S
import qualified Data.Text as T
import qualified Data.Text.Lens as T
import qualified Data.Text.IO as T
import qualified Data.Yaml as Y
import           System.Directory
import           System.FilePath.Lens
import           System.Process
import           Text.Trifecta (failed, raiseErr)


import qualified Formura.Annotation as A
import           Formura.Annotation.Boundary
import           Formura.Annotation.Representation
import           Formura.Compiler
import           Formura.CommandLineOption
import           Formura.Geometry
import           Formura.GlobalEnvironment
import           Formura.Language.Combinator (subFix)
import           Formura.NumericalConfig
import           Formura.OrthotopeMachine.Graph
import           Formura.Syntax
import           Formura.Vec
import           Formura.MPICxx.Cut hiding (cut)

showC :: Show a => a -> T.Text
showC = T.pack . show

parens :: T.Text -> T.Text
parens x = "(" <> x <> ")"

brackets :: T.Text -> T.Text
brackets x = "[" <> x <> "]"

braces :: T.Text -> T.Text
braces x = "{" <> x <> "}"

newtype VariableName = VariableName T.Text


-- | The struct for generating unique names, and holds already given names.
data NamingState = NamingState
  { _alreadyGivenNames :: S.Set T.Text
  , _alreadyGivenLocalNames :: S.Set T.Text
  , _alreadyDeclaredResourceNames :: S.Set T.Text
  , _freeNameCounter :: Integer
  , _freeLocalNameCounter :: Integer
  , _nodeIDtoLocalName :: M.Map MMNodeID T.Text
  , _loopIndexNames :: Vec T.Text
  , _loopIndexOffset :: Vec Int
  , _loopExtentNames :: Vec T.Text
  }
makeClassy ''NamingState

defaultNamingState = NamingState
  { _alreadyGivenNames = S.empty
  , _alreadyGivenLocalNames = S.empty
  , _alreadyDeclaredResourceNames = S.empty
  , _freeNameCounter = 0
  , _freeLocalNameCounter = 0
  , _nodeIDtoLocalName = M.empty
  , _loopIndexNames = PureVec ""
  , _loopIndexOffset = 0
  , _loopExtentNames = PureVec ""
  }

type MPIPlanSelector = Bool

data TranState = TranState
  { _tranSyntacticState :: CompilerSyntacticState
  , _tsNumericalConfig :: NumericalConfig
  , _tsNamingState :: NamingState
  , _theProgram :: Program
  , _theMMProgram :: MMProgram
  , _theGraph :: MMGraph
  , _tsMPIPlanSelection :: MPIPlanSelector
  , _tsMPIPlanMap :: M.Map MPIPlanSelector MPIPlan
  , _tsCommonStaticBox :: Box
  , _tsCxxTemplateWithMacro :: T.Text
  }
makeClassy ''TranState


instance HasCompilerSyntacticState TranState where
  compilerSyntacticState = tranSyntacticState
instance HasNumericalConfig TranState where
  numericalConfig = tsNumericalConfig
instance HasMachineProgram TranState MMInstruction OMNodeType where
  machineProgram = theMMProgram
instance HasNamingState TranState where
  namingState = tsNamingState
instance HasMPIPlan TranState where
  mPIPlan =
    let
      gettr s = fromJust $ M.lookup (s^.tsMPIPlanSelection) (s^.tsMPIPlanMap)
      settr s a = s & tsMPIPlanMap %~ M.insert (s^.tsMPIPlanSelection) a
    in lens gettr settr

data CProgram = CProgram { _headerFileContent :: T.Text, _sourceFileContent :: T.Text,
                         _auxFilesContent :: M.Map FilePath T.Text}
                deriving (Eq, Ord, Show)
makeLenses ''CProgram

tellH :: (MonadWriter CProgram m) => T.Text -> m ()
tellH txt = tell $ CProgram txt "" M.empty
tellC :: (MonadWriter CProgram m) => T.Text -> m ()
tellC txt = tell $ CProgram "" txt M.empty
tellBoth :: (MonadWriter CProgram m) => T.Text -> m ()
tellBoth txt = tell $ CProgram txt txt  M.empty
tellF :: (MonadWriter CProgram m) => FilePath -> T.Text -> m ()
tellF fn txt = tell $ CProgram "" ""  (M.singleton fn txt)

tellHLn :: (MonadWriter CProgram m) => T.Text -> m ()
tellHLn txt = tellH $ txt <> "\n"
tellCLn :: (MonadWriter CProgram m) => T.Text -> m ()
tellCLn txt = tellC $ txt <> "\n"
tellBothLn :: (MonadWriter CProgram m) => T.Text -> m ()
tellBothLn txt = tellBoth $ txt <> "\n"
tellFLn :: (MonadWriter CProgram m) => FilePath -> T.Text -> m ()
tellFLn fn txt = tellF fn $ txt <> "\n"


instance Monoid CProgram where
  mempty = CProgram "" "" M.empty
  mappend (CProgram h1 c1 f1) (CProgram h2 c2 f2) = CProgram (h1 <> h2) (c1 <> c2) (M.unionWith (<>) f1 f2)


type TranM = CompilerMonad GlobalEnvironment CProgram TranState

-- * Parallel code generation

-- | generate new free global name based on given identifier,
--   and prevent further generation of that name
genFreeName :: IdentName -> TranM T.Text
genFreeName = genFreeName' True

-- | generate new free local name based on given identifier,
--   and prevent further generation of that name within current scope
genFreeLocalName :: IdentName -> TranM T.Text
genFreeLocalName = genFreeName' False

-- | base function for giving names
genFreeName' :: Bool -> IdentName -> TranM T.Text
genFreeName' isGlobal ident = do
  aggNames <- use alreadyGivenNames
  aglNames <- use alreadyGivenLocalNames
  let initName = T.pack ident
      agNames = aggNames <> aglNames
      nCounter :: Lens' TranState Integer
      nCounter = if isGlobal then freeNameCounter else freeLocalNameCounter
      go = do
        ctr <- use nCounter
        let tmpName = initName <> "_" <> showC ctr
        if S.member tmpName agNames
          then (nCounter += 1) >> go
          else return tmpName
  givenName <- if S.member initName agNames then go else return initName
  (if isGlobal then alreadyGivenNames else alreadyGivenLocalNames) %= S.insert givenName
  return givenName

-- | read all numerical config from the Formura source program
setNumericalConfig :: WithCommandLineOption => TranM ()
setNumericalConfig = do
  dim <- view dimension
  ivars <- view axesNames

  mnc <- liftIO $ Y.decodeFile ncFilePath
  nc <- case mnc of
     Nothing -> raiseErr $ failed $ "cannot parse numerical config .yaml file: " ++ show ncFilePath
     Just x -> return x

  tsNumericalConfig .= nc


  when (length (nc ^. ncMPIGridShape) /= dim) $
    raiseErr $ failed $ "mpi_grid_shape need exactly " ++ show dim ++ " elements."
  when (length (nc ^. ncIntraNodeShape) /= dim) $
    raiseErr $ failed $ "intra_node_shape need exactly " ++ show dim ++ " elements."
  return ()

-- | prepare unique name for everyone
setNamingState :: TranM ()
setNamingState = do
  stateVars <- use omStateSignature
  alreadyGivenNames .= (S.fromList $ map T.pack $ M.keys stateVars)

  ans <- view axesNames
  lins <- traverse (genFreeName . ("i"++)) ans
  loopIndexNames .= lins

  luns <- traverse (genFreeName . ("N"++) . map toUpper) ans
  loopExtentNames .= luns

  let nameNode :: MMNode -> TranM MMNode
      nameNode nd = do
        let initName = case A.viewMaybe nd  of
                        Just (SourceName n) -> n
                        _                   -> "g"
        cName <- genFreeName initName
        return $ nd & A.annotation %~ A.set (VariableName cName)

  gr <- use omInitGraph
  gr2 <- flip traverse gr $ nameNode
  omInitGraph .= gr2

  gr <- use omStepGraph
  gr2 <- flip traverse gr $ nameNode
  omStepGraph .= gr2


-- | Generate C type declaration for given language.
genTypeDecl :: IdentName -> TypeExpr -> TranM T.Text
genTypeDecl name typ = case typ of
  ElemType "void" -> return ""
  ElemType "Rational" -> return $ "double " <> T.pack name
  ElemType x -> return $ T.pack  x <> " " <> T.pack name
  GridType _ x -> do
    body <- genTypeDecl name x
    if body == "" then return ""
      else do
        sz <- use ncIntraNodeShape
        let szpt = foldMap (brackets . showC) sz
        return $ body <> szpt
  _ -> raiseErr $ failed $ "Cannot translate type to C: " ++ show typ


elemTypeOfResource :: ResourceT a b -> TranM TypeExpr
elemTypeOfResource (ResourceStatic sname _) = do
  ssMap <- use omStateSignature
  let Just typ = M.lookup sname ssMap
  case typ of
    ElemType _ -> return typ
    GridType _ etyp -> return etyp
elemTypeOfResource (ResourceOMNode nid _) = do
  mmProg <- use omStepGraph
  let Just nd = M.lookup nid mmProg
  case nd ^.nodeType of
    ElemType x -> return $ ElemType x
    GridType _ etyp -> return $ subFix etyp

tellMPIRequestDecl :: T.Text -> TranM ()
tellMPIRequestDecl name = do
  adrn <- use alreadyDeclaredResourceNames
  case S.member name adrn of
    True -> return ()
    False -> do
      alreadyDeclaredResourceNames %= S.insert name
      tellH "extern "
      tellBothLn $ "MPI_Request "<>name<>";\n"
tellResourceDecl :: T.Text -> ResourceT a b -> Box -> TranM ()
tellResourceDecl = tellResourceDecl' False

tellResourceDecl' :: Bool -> T.Text -> ResourceT a b -> Box -> TranM ()
tellResourceDecl' isInClass name rsc box0 = do
  adrn <- use alreadyDeclaredResourceNames
  case S.member name adrn || name == "" of
    True -> return ()
    False -> do
      alreadyDeclaredResourceNames %= S.insert name

      typ <- elemTypeOfResource rsc
      let szpt = foldMap (brackets . showC) sz
          sz = box0 ^.upperVertex - box0 ^. lowerVertex

      decl <- case typ of
        ElemType "void" -> return ""
        ElemType "Rational" -> return $ "double " <> name <> szpt
        ElemType x -> return $ T.pack  x <> " " <> name <> szpt
        _ -> raiseErr $ failed $ "Cannot translate type to C: " ++ show typ
      when (decl /= "") $ do

        when isInClass $ do
          tellH decl
          tellHLn ";"
        when (not isInClass) $ do
          tellH "extern "
          tellBoth decl
          tellBothLn ";"


--       case rsc of
--         ResourceStatic _ _ -> do
--             tellH "extern "
--             tellBoth decl
--             tellBothLn ";"
--         _ -> do
--           when (decl /= "") $ tellCLn $ decl <> ";"


tellFacetDecl :: FacetID -> [RidgeID] -> TranM ()
tellFacetDecl f rs = do
  let name = T.pack $ toCName f
  tellH $ "struct " <> name <> "{"

  ralloc <- use planRidgeAlloc

  forM_ rs $ \rk -> do
    name <- nameRidgeResource' True rk SendRecv
    let Just box0 = M.lookup rk ralloc
    tellResourceDecl' True name (rk ^. ridgeDelta) box0

  tellH "};"

  tellH $ "extern struct " <> name <> " " <> name <> "_Send;"
  tellH $ "extern struct " <> name <> " " <> name <> "_Recv;"
  tellC $ "struct " <> name <> " " <> name <> "_Send;"
  tellC $ "struct " <> name <> " " <> name <> "_Recv;"
  return ()


toCName :: Show a => a -> IdentName
toCName a = postfix $ fix $ go False $ prefix $ show a
  where
    go _ [] = []
{'fraction_non_alphanumeric': 0.10252416510147053, 'fraction_numerical': 0.0025195840395803746, 'mean_word_length': 2.801811215604319, 'pattern_counts': {'":': 0, '<': 148, '<?xml version=': 0, '>': 235, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6063288', 'n_tokens_mistral': 7116, 'n_tokens_neox': 6698, 'n_words': 2134}