<commit_before>module L0C.FirstOrderTransform
  ( transformProg )
  where

import Control.Monad.State

import Data.Loc

import L0C.L0
import L0C.FreshNames

type TransformM = State VNameSource

-- | Return a new, fresh name, with the given string being part of the
-- name.
new :: String -> TransformM VName
new = state . newVName

transformProg :: Prog -> Prog
transformProg prog =
  Prog $ runTransformM $ mapM transformFunDec $ progFunctions prog
  where runTransformM m = evalState m $ newNameSourceForProg prog

transformFunDec :: FunDec -> TransformM FunDec
transformFunDec (fname, rettype, params, body, loc) = do
  body' <- transformExp body
  return (fname, rettype, params, body', loc)

transformExp :: Exp -> TransformM Exp

transformExp mape@(Map fun e intype loc) = do
  -- We have to allocate a new array up front.  This is a bit tricky,
  -- as in case the new array is an array-of-arrays, we need to
  -- compute the first element in order to get the proper size.  We
  -- evaluate the function on the first element of the array and use
  -- that to construct the array for the rest.  If the input array is
  -- empty, we simply return an empty output array.
  (inarr, inarrv, inarrlet) <- newLet e "inarr"
  (i, iv) <- newVar "i" (Elem Int) loc
  (_, nv, nlet) <- newLet (Size inarrv loc) "n"
  let zero = Literal (IntVal 0) loc
      index0 = Index inarr [zero] intype loc
      index = Index inarr [iv] intype loc
  funcall0 <- transformLambda fun [index0]
  funcall <- transformLambda fun [index]
  (outarr, outarrv, outarrlet) <- newLet (Replicate nv funcall0 loc) "outarr"
  let branch = If (BinOp Less zero nv (Elem Bool) loc)
               (outarrlet letbody)
               (maybeCopy $ Literal (blankValue $ typeOf mape) loc)
               (typeOf mape) loc
      letbody = DoLoop (Id outarr) outarrv i nv loopbody outarrv loc
      loopbody = LetWith outarr outarr [iv] funcall outarrv loc
  return $ inarrlet $ nlet branch

transformExp (Reduce fun accexp arrexp intype loc) = do
  ((arr, arrv), (acc, accv), (i, iv), redlet) <- newReduction loc arrexp accexp
  let index = Index arr [iv] intype loc
  funcall <- transformLambda fun [accv, index]
  let loop = DoLoop (Id acc) accv i (Size arrv loc) loopbody accv loc
      loopbody = LetPat (Id acc) funcall accv loc
  return $ redlet loop

transformExp (Scan fun accexp arrexp intype loc) = do
  ((arr, arrv), (acc, accv), (i, iv), redlet) <- newReduction loc arrexp accexp
  let index = Index arr [iv] intype loc
  funcall <- transformLambda fun [accv, index]
  let loop = DoLoop (TupId [Id acc, Id arr] loc) (TupLit [accv, arrv] loc) i (Size arrv loc) loopbody arrv loc
      loopbody = LetWith arr arr [iv] funcall (TupLit [index, arrv] loc) loc
  return $ redlet loop

transformExp (Filter fun arrexp rowtype loc) = do
  (arr, arrv, arrlet) <- newLet arrexp "arr"
  (_, nv, nlet) <- newLet (Size arrv loc) "n"
  let checkempty nonempty = If (BinOp Equal nv (intval 0) (Elem Bool) loc)
                            (Literal (emptyArray rowtype) loc) nonempty
                            (typeOf arrexp) loc
  (x, xv) <- newVar "x" rowtype loc
  (i, iv) <- newVar "i" (Elem Int) loc
  fun' <- transformLambda fun [xv]
  let branch = If fun' (intval 1) (intval 0) (Elem Int) loc
      indexin0 = Index arr [intval 0] rowtype loc
      indexin = Index arr [iv] rowtype loc
  mape <- transformExp $ Map (AnonymFun [x] branch (Elem Int) loc) arrv rowtype loc
  plus <- do
    (a,av) <- newVar "a" (Elem Int) loc
    (b,bv) <- newVar "b" (Elem Int) loc
    return $ AnonymFun [a, b] (BinOp Plus av bv (Elem Int) loc) (Elem Int) loc
  scan <- transformExp $ Scan plus (intval 0) mape (Elem Int) loc
  (ia, _, ialet) <- newLet scan "ia"
  let indexia ind = Index ia [ind] (Elem Int) loc
      indexiaend = indexia (BinOp Minus nv (intval 1) (Elem Int) loc)
      indexi = indexia iv
      indexim1 = indexia (BinOp Minus iv (intval 1) (Elem Int) loc)
  (res, resv, reslet) <- newLet (Replicate indexiaend indexin0 loc) "res"
  let loop = DoLoop (Id res) resv i nv loopbody resv loc
      loopbody = If (Or (BinOp Equal iv (intval 0) (Elem Bool) loc)
                        (And (BinOp Less (intval 0) iv (Elem Bool) loc)
                             (BinOp Equal indexi indexim1 (Elem Bool) loc) loc)
                     loc)
                 resv update (typeOf resv) loc
      update = LetWith res res [BinOp Minus indexi (intval 1) (Elem Int) loc] indexin resv loc
  return $ arrlet $ nlet $ checkempty $ ialet $ reslet loop
  where intval x = Literal (IntVal x) loc

transformExp (Mapall fun arrexp loc) = transformExp =<< toMap arrexp
  where toMap e = case peelArray 1 $ typeOf e of
                    Just et -> do
                      (x,xv) <- newVar "x" et loc
                      body <- toMap xv
                      let ot = arrayType (arrayDims et) (typeOf fun) Nonunique
                      return $ Map (AnonymFun [x] body ot loc) e et loc
                    _ -> transformLambda fun [e]

transformExp (Redomap redfun mapfun accexp arrexp _ loc) = do
  ((arr, arrv), (acc, accv), (i, iv), redlet) <- newReduction loc arrexp accexp
  let index = Index arr [iv] (stripArray 1 $ typeOf arrexp) loc
  mapfuncall <- transformLambda mapfun [index]
  redfuncall <- transformLambda redfun [accv, mapfuncall]
  let loop = DoLoop (Id acc) accv i (Size arrv loc) loopbody accv loc
      loopbody = LetWith acc acc [] redfuncall accv loc
  return $ redlet loop

transformExp e = mapExpM transform e
  where transform = identityMapper {
                      mapOnExp = transformExp
                    }

newReduction :: SrcLoc -> Exp -> Exp
             -> TransformM ((Ident, Exp),
                            (Ident, Exp),
                            (Ident, Exp),
                            Exp -> Exp)
newReduction loc arrexp accexp = do
  (arr, arrv, arrlet) <- newLet arrexp "arr"
  (acc, accv, acclet) <- newLet accexp "acc"
  (i, iv) <- newVar "i" (Elem Int) loc
  return ((arr, arrv), (acc, accv), (i, iv), acclet . arrlet)

newLet :: Exp -> String -> TransformM (Ident, Exp, Exp -> Exp)
newLet e name = do
  e' <- liftM maybeCopy $ transformExp e
  (x,xv) <- newVar name (typeOf e') loc
  let xlet body = LetPat (Id x) e' body loc
  return (x, xv, xlet)
  where loc = srclocOf e


newVar :: String -> Type -> SrcLoc -> TransformM (Ident, Exp)
newVar name tp loc = do
  x <- new name
  return (Ident x tp loc, Var $ Ident x tp loc)

-- | @maybeCopy e@ returns a copy expression containing @e@ if @e@ is
-- not unique or a basic type, otherwise just returns @e@ itself.
maybeCopy :: Exp -> Exp
maybeCopy e
  | unique e || basicType (typeOf e)  = e
  | otherwise = Copy e $ srclocOf e

transformLambda :: Lambda -> [Exp] -> TransformM Exp
transformLambda (AnonymFun params body _ loc) args = do
  body' <- transformExp body
  return $ foldl bind body' $ zip params args
  where bind e (Ident pname ptype _, arg) = LetPat (Id $ Ident pname ptype loc) arg e loc
transformLambda (CurryFun fname curryargs rettype loc) args = do
  curryargs' <- mapM transformExp curryargs
  return $ Apply fname (curryargs'++args) rettype loc<commit_msg>Handle map2 and friends in first-order-transform.<commit_after>-- | The code generator cannot handle the array combinators (@map@ and
-- friends), so this module was written to transform them into the
-- equivalent do-loops.  The transformation is currently rather naive
-- - it's certainly worth considering when we can express such
-- transformations in-place.  This module should be run very late in
-- the compilation pipeline, ideally just before the code generator.
module L0C.FirstOrderTransform
  ( transformProg )
  where

import Control.Applicative
import Control.Monad.State

import Data.Loc

import L0C.L0
import L0C.FreshNames

type TransformM = State VNameSource

-- | Return a new, fresh name, with the given string being part of the
-- name.
new :: String -> TransformM VName
new = state . newVName

-- | Perform the first-order transformation on an L0 program.
transformProg :: Prog -> Prog
transformProg prog =
  Prog $ runTransformM $ mapM transformFunDec $ progFunctions prog
  where runTransformM m = evalState m $ newNameSourceForProg prog

transformFunDec :: FunDec -> TransformM FunDec
transformFunDec (fname, rettype, params, body, loc) = do
  body' <- transformExp body
  return (fname, rettype, params, body', loc)

-- We sometimes have to allocate new arrays up front.  This is a bit
-- tricky, as in case the new array is multidimensional, we need to
-- compute the first element in order to get the proper size.  For
-- example, in the case for map, we evaluate the function on the first
-- element of the array and use that to construct the array for the
-- rest.  If the input array is empty, we simply return an empty
-- output array.
transformExp :: Exp -> TransformM Exp

transformExp mape@(Map fun e intype loc) =
  newLet "inarr" e $ \inarr inarrv -> do
    (i, iv) <- newVar loc "i" (Elem Int)
    newLet "n" (Size inarrv loc) $ \_ nv -> do
      let zero = Literal (IntVal 0) loc
          index0 = Index inarr [zero] intype loc
          indexi = Index inarr [iv] intype loc
      funcall0 <- transformLambda fun [index0]
      funcall <- transformLambda fun [indexi]
      newLet "outarr" (Replicate nv funcall0 loc) $ \outarr outarrv -> do
        let branch = If (BinOp Less zero nv (Elem Bool) loc)
                     letbody
                     (maybeCopy $ Literal (blankValue $ typeOf mape) loc)
                     (typeOf mape) loc
            letbody = DoLoop (Id outarr) outarrv i nv loopbody outarrv loc
            loopbody = LetWith outarr outarr [iv] funcall outarrv loc
        return branch

transformExp (Reduce fun accexp arrexp intype loc) =
  newReduction loc arrexp accexp $ \(arr, arrv) (acc, accv) (i, iv) -> do
    let indexi = Index arr [iv] intype loc
    funcall <- transformLambda fun [accv, indexi]
    let loop = DoLoop (Id acc) accv i (Size arrv loc) loopbody accv loc
        loopbody = LetPat (Id acc) funcall accv loc
    return loop

transformExp (Scan fun accexp arrexp intype loc) =
  newReduction loc arrexp accexp $ \(arr, arrv) (acc, accv) (i, iv) -> do
    let indexi = Index arr [iv] intype loc
    funcall <- transformLambda fun [accv, indexi]
    let loop = DoLoop (TupId [Id acc, Id arr] loc) (TupLit [accv, arrv] loc) i (Size arrv loc) loopbody arrv loc
        loopbody = LetWith arr arr [iv] funcall (TupLit [indexi, arrv] loc) loc
    return loop

transformExp (Filter fun arrexp rowtype loc) =
  newLet "arr" arrexp $ \arr arrv ->
    newLet "n" (Size arrv loc) $ \_ nv -> do
      let checkempty nonempty = If (BinOp Equal nv (intval 0) (Elem Bool) loc)
                                (Literal (emptyArray rowtype) loc) nonempty
                                (typeOf arrexp) loc
      (x, xv) <- newVar loc "x" rowtype
      (i, iv) <- newVar loc "i" (Elem Int)
      fun' <- transformLambda fun [xv]
      let branch = If fun' (intval 1) (intval 0) (Elem Int) loc
          indexin0 = Index arr [intval 0] rowtype loc
          indexin = Index arr [iv] rowtype loc
      mape <- transformExp $
              Map (AnonymFun [x] branch (Elem Int) loc) arrv rowtype loc
      plus <- do
        (a,av) <- newVar loc "a" (Elem Int)
        (b,bv) <- newVar loc "b" (Elem Int)
        return $ AnonymFun [a, b] (BinOp Plus av bv (Elem Int) loc) (Elem Int) loc
      scan <- transformExp $ Scan plus (intval 0) mape (Elem Int) loc
      newLet "ia" scan $ \ia _ -> do
        let indexia ind = Index ia [ind] (Elem Int) loc
            indexiaend = indexia (BinOp Minus nv (intval 1) (Elem Int) loc)
            indexi = indexia iv
            indexim1 = indexia (BinOp Minus iv (intval 1) (Elem Int) loc)
        newLet "res" (Replicate indexiaend indexin0 loc) $ \res resv -> do
          let loop = DoLoop (Id res) resv i nv loopbody resv loc
              loopbody = If (Or (BinOp Equal iv (intval 0) (Elem Bool) loc)
                                (And (BinOp Less (intval 0) iv (Elem Bool) loc)
                                     (BinOp Equal indexi indexim1 (Elem Bool) loc) loc)
                             loc)
                         resv update (typeOf resv) loc
              update = LetWith res res [BinOp Minus indexi (intval 1) (Elem Int) loc]
                       indexin resv loc
          return $ checkempty loop
        where intval x = Literal (IntVal x) loc

transformExp (Mapall fun arrexp loc) = transformExp =<< toMap arrexp
  where toMap e = case peelArray 1 $ typeOf e of
                    Just et -> do
                      (x,xv) <- newVar loc "x" et
                      body <- toMap xv
                      let ot = arrayType (arrayDims et) (typeOf fun) Nonunique
                      return $ Map (AnonymFun [x] body ot loc) e et loc
                    _ -> transformLambda fun [e]

transformExp (Redomap redfun mapfun accexp arrexp _ loc) =
  newReduction loc arrexp accexp $ \(arr, arrv) (acc, accv) (i, iv) -> do
    let indexi = Index arr [iv] (stripArray 1 $ typeOf arrexp) loc
    mapfuncall <- transformLambda mapfun [indexi]
    redfuncall <- transformLambda redfun [accv, mapfuncall]
    let loop = DoLoop (Id acc) accv i (Size arrv loc) loopbody accv loc
        loopbody = LetWith acc acc [] redfuncall accv loc
    return loop

transformExp mape@(Map2 fun arrs _ loc) = do
  let zero = Literal (IntVal 0) loc
  newLets "inarr" arrs $ \inarrs _ -> do
    (i, iv) <- newVar loc "i" (Elem Int)
    newLet "n" (size inarrs) $ \_ nv -> do
      funcall0 <- transformLambda fun $ index inarrs zero
      funcall <- transformLambda fun $ index inarrs iv
      newResultArray nv funcall0 $ \outarr outarrv -> do
        loopbody <- letwith outarr iv funcall outarrv
        let branch = If (BinOp Less zero nv (Elem Bool) loc)
                     letbody
                     (maybeCopy $ Literal (blankValue $ typeOf mape) loc)
                     (typeOf mape) loc
            letbody = DoLoop (pattern outarr loc) outarrv i nv loopbody
                      outarrv loc
        return branch

transformExp (Reduce2 fun accexp arrexps _ loc) =
  newReduction2 loc arrexps accexp $ \(arr, _) (acc, accv) (i, iv) -> do
    funcall <- transformLambda fun $ map Var acc ++ index arr iv
    let loop = DoLoop (pattern acc loc) accv i (size arr)
               funcall accv loc
    return loop

transformExp (Scan2 fun accexp arrexps _ loc) =
  newReduction2 loc arrexps accexp $ \(arr, arrv) (acc, accv) (i, iv) -> do
    funcall <- transformLambda fun $ map Var acc ++ index arr iv
    loopbody <- letwith arr iv funcall (TupLit (accv:map Var arr) loc)
    let looppat = case pattern acc loc of
                    Id k         -> TupId (Id k : map Id arr) loc
                    TupId pats _ -> TupId (pats ++ map Id arr) loc
        loop = DoLoop looppat (TupLit (map Var $ acc ++ arr) loc)
               i (size arr)
               loopbody arrv loc
    return loop

transformExp filtere@(Filter2 fun arrexps loc) =
  newLets "arr" arrexps $ \arr _ ->
    newLet "n" (size arr) $ \_ nv -> do
      let checkempty nonempty =
            If (BinOp Equal nv (intval 0) (Elem Bool) loc)
            (Literal (blankValue $ typeOf filtere) loc) nonempty
            (typeOf filtere) loc
          rowtypes = map (rowType . typeOf) arr
      (xs, _) <- unzip <$> mapM (newVar loc "x") rowtypes
      (i, iv) <- newVar loc "i" $ Elem Int
      fun' <- transformLambda fun $ map Var xs
      let branch = If fun' (intval 1) (intval 0) (Elem Int) loc
          indexin0 = index arr $ intval 0
          indexin = index arr iv
      mape <- transformExp $
              Map2 (AnonymFun xs branch (Elem Int) loc) (map Var arr)
              (Elem $ Tuple rowtypes) loc
      plus <- do
        (a,av) <- newVar loc "a" (Elem Int)
        (b,bv) <- newVar loc "b" (Elem Int)
        return $ AnonymFun [a, b] (BinOp Plus av bv (Elem Int) loc) (Elem Int) loc
      scan <- transformExp $ Scan2 plus (intval 0) [mape] (Elem Int) loc
      newLet "ia" scan $ \ia _ -> do
        let indexia ind = Index ia [ind] (Elem Int) loc
            indexiaend = indexia (sub1 nv)
            indexi = indexia iv
            indexim1 = indexia (sub1 iv)
        newResultArray indexiaend (TupLit indexin0 loc) $ \res resv -> do
          update <- letwith res (sub1 indexi) (TupLit indexin loc) resv
          let loop = DoLoop (pattern res loc) resv i nv loopbody resv loc
              loopbody = If (Or (BinOp Equal iv (intval 0) (Elem Bool) loc)
                                (And (BinOp Less (intval 0) iv (Elem Bool) loc)
                                     (BinOp Equal indexi indexim1 (Elem Bool) loc) loc)
                             loc)
                         resv update (typeOf resv) loc
          return $ checkempty loop
  where intval x = Literal (IntVal x) loc
        sub1 e = BinOp Minus e (intval 1) (Elem Int) loc

transformExp (Mapall2 fun arrexps loc) = transformExp =<< toMap arrexps
  where toMap es =
          case mapM (peelArray 1 . typeOf) es of
            Just rts -> do
              (xs,_) <- unzip <$> mapM (newVar loc "x") rts
              body <- toMap $ map Var xs
              return $ Map2 (AnonymFun xs body (typeOf body) loc) es (Elem $ Tuple rts) loc
            _ -> transformLambda fun es

transformExp (Redomap2 redfun mapfun accexp arrexps _ loc) =
  newReduction2 loc arrexps accexp $ \(arr, _) (acc, accv) (i, iv) -> do
    mapfuncall <- transformLambda mapfun $ index arr iv
    let loop loopbody = DoLoop (pattern acc loc) accv i (size arr) loopbody accv loc
    case typeOf mapfuncall of
      Elem (Tuple ts) -> do
        names <- mapM (liftM fst . newVar loc "mapres") ts
        redfuncall <- transformLambda redfun $ map Var $ acc ++ names
        return $ loop $ LetPat (TupId (map Id names) loc) mapfuncall redfuncall loc
      t -> do
        name <- fst <$> newVar loc "mapres" t
        redfuncall <- transformLambda redfun $ map Var $ acc ++ [name]
        return $ loop $ LetPat (Id name) mapfuncall redfuncall loc

transformExp e = mapExpM transform e
  where transform = identityMapper {
                      mapOnExp = transformExp
                    }

newReduction :: SrcLoc -> Exp -> Exp
             -> ((Ident, Exp) -> (Ident, Exp) -> (Ident, Exp) -> TransformM Exp)
             -> TransformM Exp
newReduction loc arrexp accexp body =
  newLet "arr" arrexp $ \arr arrv ->
    newLet "acc" accexp $ \acc accv -> do
      (i, iv) <- newVar loc "i" (Elem Int)
      body (arr, arrv) (acc, accv) (i, iv)

newReduction2 :: SrcLoc -> [Exp] -> Exp
             -> (([Ident], Exp) -> ([Ident], Exp) -> (Ident, Exp) -> TransformM Exp)
             -> TransformM Exp
newReduction2 loc arrexps accexp body = do
  (i, iv) <- newVar loc "i" (Elem Int)
  newLets "arr" arrexps $ \arr arrv ->
    case typeOf accexp of
      Elem (Tuple ets) -> do
        (names, namevs) <- unzip <$> mapM (newVar loc "acc") ets
        let binder inner = LetPat (TupId (map Id names) loc) accexp inner loc
        binder <$> body (arr, arrv) (names, TupLit namevs loc) (i, iv)
      t -> do (acc, accv) <- newVar loc "acc" t
              let binder inner = LetPat (Id acc) accexp inner loc
              binder <$> body (arr, arrv) ([acc], accv) (i, iv)

newLet :: String -> Exp -> (Ident -> Exp -> TransformM Exp)
       -> TransformM Exp
newLet name e body = do
  e' <- liftM maybeCopy $ transformExp e
  (x,xv) <- newVar loc name (typeOf e')
  let xlet inner = LetPat (Id x) e' inner loc
  xlet <$> body x xv
  where loc = srclocOf e

newLets :: String -> [Exp] -> ([Ident] -> Exp -> TransformM Exp)
        -> TransformM Exp
newLets k es body = newLets' es []
  where newLets' [] [name]      = body [name] $ Var name
        newLets' [] names       =
          body (reverse names) $ TupLit (map Var $ reverse names) noLoc
        newLets' (e:rest) names =
          newLet k e $ \name _ -> newLets' rest (name:names)

newVar :: SrcLoc -> String -> Type -> TransformM (Ident, Exp)
newVar loc name tp = do
  x <- new name
  return (Ident x tp loc, Var $ Ident x tp loc)

-- | @maybeCopy e@ returns a copy expression containing @e@ if @e@ is
-- not unique or a basic type, otherwise just returns @e@ itself.
maybeCopy :: Exp -> Exp
maybeCopy e
  | unique e || basicType (typeOf e)  = e
  | otherwise = Copy e $ srclocOf e

index :: [Ident] -> Exp -> [Exp]
index arrs i = flip map arrs $ \arr ->
               Index arr [i] (stripArray 1 $ identType arr) $ srclocOf i

newResultArray :: Exp -> Exp -> ([Ident] -> Exp -> TransformM Exp) -> TransformM Exp
newResultArray sizeexp valueexp body =
  case typeOf valueexp of
    Elem (Tuple ets) -> do
      (names, namevs) <- unzip <$> mapM (newVar loc "outarr_v") ets
      let bnd inner = LetPat (TupId (map Id names) loc ) valueexp inner loc
          arrexp = [ maybeCopy $ Replicate sizeexp namev loc
                     | namev <- namevs ]
      bnd <$> newLets "outarr" arrexp body
    _ -> do let rep = maybeCopy $ Replicate sizeexp valueexp loc
            (name, namev) <- newVar loc "outarr" $ typeOf rep
            let bnd inner = LetPat (Id name) rep inner loc
            bnd <$> body [name] namev
  where loc = srclocOf valueexp

letwith :: [Ident] -> Exp -> Exp -> Exp -> TransformM Exp
letwith ks i v body =
  case typeOf v of
    Elem (Tuple ets) -> do
      names <- mapM (liftM fst . newVar loc "tup") ets
      let comb inner (k, name) = LetWith k k [i] (Var name) inner loc
      return $ LetPat (TupId (map Id names) loc) v
               (foldl comb body $ zip ks names) loc
    _ -> do let comb inner k = LetWith k k [i] v inner loc
            return $ foldl comb body ks
  where loc = srclocOf body

pattern :: [Ident] -> SrcLoc -> TupIdent
pattern [k] _ = Id k
pattern ks loc = TupId (map Id ks) loc

size :: [Ident] -> Exp
size [] = Literal (IntVal 0) noLoc
size (k:_) = Size (Var k) $ srclocOf k

transformLambda :: Lambda -> [Exp] -> TransformM Exp
transformLambda (AnonymFun params body _ loc) args = do
  body' <- transformExp body
  return $ foldl bind body' $ zip params args
  where bind e (Ident pname ptype _, arg) = LetPat (Id $ Ident pname ptype loc) arg e loc
transformLambda (CurryFun fname curryargs rettype loc) args = do
  curryargs' <- mapM transformExp curryargs
  return $ Apply fname (curryargs'++args) rettype loc
{'fraction_non_alphanumeric': 0.0969447182152137, 'fraction_numerical': 0.0039422990771436255, 'mean_word_length': 2.3293064876957494, 'pattern_counts': {'":': 0, '<': 109, '<?xml version=': 0, '>': 119, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7462506', 'n_tokens_mistral': 7371, 'n_tokens_neox': 7076, 'n_words': 2905}