<commit_before>{-# LANGUAGE OverloadedStrings #-}

module Main where

import qualified Data.Foldable as F
import qualified Data.Text     as T
import           Turtle

import           Utils         as U

main :: IO ()
main = do
  echo $ "Please provide the path to the HTML files:"
  Just filesPath <- readline
  let textFilesPath = fromText filesPath
  if T.length filesPath > 0 then do
    files <- U.filePathToText <$> U.fileList textFilesPath
    let ret = T.replicate 60 $ T.pack "-"
    echo $ ret
    F.forM_ files $ \file -> do
      echo $ format (w%"\n") file
      result <- U.report $ T.unpack $ file
      F.forM_ result $ \tag -> do
        echo $ format ("-> "%s) tag
      echo $ ret
    else echo "Empty path - exit"<commit_msg>Updated output rendering.<commit_after>{-# LANGUAGE OverloadedStrings #-}

module Main where

import qualified Data.Foldable as F
import qualified Data.Text     as T
import           Turtle

import           Utils         as U

main :: IO ()
main = do
  echo $ "Please provide the path to the HTML files:"
  Just filesPath <- readline
  let textFilesPath = fromText filesPath
  if T.length filesPath > 0 then do
    files <- U.filePathToText <$> U.fileList textFilesPath
    let ret = T.replicate 30 $ T.pack "- "
    echo $ ret
    F.forM_ files $ \file -> do
      echo $ format (w%"\n") file
      result <- U.report $ T.unpack $ file
      echo $ format ("Found "%d%" data attribute(s)") (length result)
      F.forM_ result $ \tag -> do
        echo $ format ("-> "%s) tag
      echo $ ret
    else echo "Empty path - exit"
{'fraction_non_alphanumeric': 0.10425937698664971, 'fraction_numerical': 0.003814367450731087, 'mean_word_length': 2.2655601659751037, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '284792', 'n_tokens_mistral': 504, 'n_tokens_neox': 486, 'n_words': 181}