<commit_before>{-# OPTIONS_GHC -O0 -fno-case-merge -fno-strictness -fno-cse #-}
{-# LANGUAGE OverloadedStrings           #-}

module Text.XmlHtml.HTML.Meta
  ( voidTags
  , rawTextTags
  , endOmittableLast
  , endOmittableNext
  , predefinedRefs
  ) where

import           Data.Monoid
import           Data.HashMap.Strict (HashMap)
import qualified Data.HashMap.Strict as M
import           Data.HashSet (HashSet)
import qualified Data.HashSet as S
import           Data.Text (Text)

------------------------------------------------------------------------------
-- Metadata used for HTML5 quirks mode.                                     --
------------------------------------------------------------------------------

{-# NOINLINE voidTags #-}
voidTags :: HashSet Text
voidTags = S.fromList [
    "area", "base", "br", "col", "command", "embed", "hr", "img", "input",
    "keygen", "link", "meta", "param", "source", "track", "wbr"
    ]

{-# NOINLINE rawTextTags #-}
rawTextTags :: HashSet Text
rawTextTags = S.fromList [ "script", "style" ]

------------------------------------------------------------------------------
-- | Tags which can be implicitly ended in case they are the last element in
-- their parent.  This list actually includes all of the elements that have
-- any kind of omittable end tag, since in general when an element with an
-- omittable end tag isn't specified to be omittable in this way, it's just
-- because in a complete document it isn't expected to ever be the last thing
-- in its parent.  We aren't interested in enforcing element structure rules,
-- so we'll allow it anyway.
{-# NOINLINE endOmittableLast #-}
endOmittableLast :: HashSet Text
endOmittableLast = S.fromList [
    "body", "colgroup", "dd", "dt", "head", "html", "li", "optgroup",
    "option", "p", "rp", "rt", "tbody", "td", "tfoot", "th", "thead", "tr"
    ]

------------------------------------------------------------------------------
-- | Tags which should be considered automatically ended in case one of a
-- certain set of tags pops up.
{-# NOINLINE endOmittableNext #-}
endOmittableNext :: HashMap Text (HashSet Text)
endOmittableNext = M.fromList [
    ("colgroup", S.fromList ["caption", "colgroup", "tbody",
                             "thead", "tfoot", "tr"]),
    ("dd",       S.fromList ["dd", "dt"]),
    ("dt",       S.fromList ["dd", "dt"]),
    ("head",     S.fromList ["body"]),
    ("li",       S.fromList ["li"]),
    ("optgroup", S.fromList ["optgroup"]),
    ("option",   S.fromList ["optgroup", "option"]),
    ("p",        S.fromList ["address", "article", "aside", "blockquote",
                             "dir", "div", "dl", "fieldset", "footer",
                             "form", "h1", "h2", "h3", "h4", "h5", "h6",
                             "header", "hgroup", "hr", "menu", "nav", "ol",
                             "p", "pre", "section", "table", "ul"]),
    ("rp",       S.fromList ["rp", "rt"]),
    ("rt",       S.fromList ["rp", "rt"]),
    ("tbody",    S.fromList ["tbody", "tfoot", "thead"]),
    ("td",       S.fromList ["td", "th"]),
    ("tfoot",    S.fromList ["tbody", "tfoot", "thead"]),
    ("th",       S.fromList ["td", "th"]),
    ("thead",    S.fromList ["tbody", "tfoot", "thead"]),
    ("tr",       S.fromList ["tr"])
    ]

{-# NOINLINE predefinedRefs #-}
predefinedRefs :: HashMap Text Text
predefinedRefs = mconcat $ map M.fromList [
      reftab1
    , reftab2
    , reftab3
    , reftab4
    , reftab5
    , reftab6
    , reftab7
    , reftab8
    , reftab9
    , reftab10
    , reftab11
    , reftab12
    , reftab13
    , reftab14
    , reftab15
    , reftab16
    , reftab17
    , reftab18
    , reftab19
    , reftab20
    , reftab21<commit_msg>Added comments to the newly exported functions<commit_after>{-# OPTIONS_GHC -O0 -fno-case-merge -fno-strictness -fno-cse #-}
{-# LANGUAGE OverloadedStrings           #-}

module Text.XmlHtml.HTML.Meta
  ( voidTags
  , rawTextTags
  , endOmittableLast
  , endOmittableNext
  , predefinedRefs
  ) where

import           Data.Monoid
import           Data.HashMap.Strict (HashMap)
import qualified Data.HashMap.Strict as M
import           Data.HashSet (HashSet)
import qualified Data.HashSet as S
import           Data.Text (Text)

------------------------------------------------------------------------------
-- Metadata used for HTML5 quirks mode.                                     --
------------------------------------------------------------------------------

------------------------------------------------------------------------------
-- | Void elements as defined by the HTML5 spec.
{-# NOINLINE voidTags #-}
voidTags :: HashSet Text
voidTags = S.fromList [
    "area", "base", "br", "col", "command", "embed", "hr", "img", "input",
    "keygen", "link", "meta", "param", "source", "track", "wbr"
    ]

------------------------------------------------------------------------------
-- | Elements that XmlHtml treats as raw text.  Raw text elements are not
-- allowed to have any other tags in them.  This is necessary to support the
-- Javascript less than operator inside a script tag, for example.  Here we
-- deviate a little from the spec because Heist users expect to be able to use
-- splices inside the title tag.
{-# NOINLINE rawTextTags #-}
rawTextTags :: HashSet Text
rawTextTags = S.fromList [ "script", "style" ]

------------------------------------------------------------------------------
-- | Tags which can be implicitly ended in case they are the last element in
-- their parent.  This list actually includes all of the elements that have
-- any kind of omittable end tag, since in general when an element with an
-- omittable end tag isn't specified to be omittable in this way, it's just
-- because in a complete document it isn't expected to ever be the last thing
-- in its parent.  We aren't interested in enforcing element structure rules,
-- so we'll allow it anyway.
{-# NOINLINE endOmittableLast #-}
endOmittableLast :: HashSet Text
endOmittableLast = S.fromList [
    "body", "colgroup", "dd", "dt", "head", "html", "li", "optgroup",
    "option", "p", "rp", "rt", "tbody", "td", "tfoot", "th", "thead", "tr"
    ]

------------------------------------------------------------------------------
-- | Tags which should be considered automatically ended in case one of a
-- certain set of tags pops up.
{-# NOINLINE endOmittableNext #-}
endOmittableNext :: HashMap Text (HashSet Text)
endOmittableNext = M.fromList [
    ("colgroup", S.fromList ["caption", "colgroup", "tbody",
                             "thead", "tfoot", "tr"]),
    ("dd",       S.fromList ["dd", "dt"]),
    ("dt",       S.fromList ["dd", "dt"]),
    ("head",     S.fromList ["body"]),
    ("li",       S.fromList ["li"]),
    ("optgroup", S.fromList ["optgroup"]),
    ("option",   S.fromList ["optgroup", "option"]),
    ("p",        S.fromList ["address", "article", "aside", "blockquote",
                             "dir", "div", "dl", "fieldset", "footer",
                             "form", "h1", "h2", "h3", "h4", "h5", "h6",
                             "header", "hgroup", "hr", "menu", "nav", "ol",
                             "p", "pre", "section", "table", "ul"]),
    ("rp",       S.fromList ["rp", "rt"]),
    ("rt",       S.fromList ["rp", "rt"]),
    ("tbody",    S.fromList ["tbody", "tfoot", "thead"]),
    ("td",       S.fromList ["td", "th"]),
    ("tfoot",    S.fromList ["tbody", "tfoot", "thead"]),
    ("th",       S.fromList ["td", "th"]),
    ("thead",    S.fromList ["tbody", "tfoot", "thead"]),
    ("tr",       S.fromList ["tr"])
    ]

------------------------------------------------------------------------------
-- | Predefined character entity references as defined by the HTML5 spec.
{-# NOINLINE predefinedRefs #-}
predefinedRefs :: HashMap Text Text
predefinedRefs = mconcat $ map M.fromList [
      reftab1
    , reftab2
    , reftab3
    , reftab4
    , reftab5
    , reftab6
    , reftab7
    , reftab8
    , reftab9
    , reftab10
    , reftab11
    , reftab12
    , reftab13
    , reftab14
    , reftab15
    , reftab16
    , reftab17
    , reftab18
    , reftab19
    , reftab20
    , reftab21
{'fraction_non_alphanumeric': 0.2490215264187867, 'fraction_numerical': 0.010273972602739725, 'mean_word_length': 1.7653026716266487, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4414379', 'n_tokens_mistral': 2529, 'n_tokens_neox': 2452, 'n_words': 747}