<commit_before>
module Text.HTML.TagSoup.Options_ where

import Text.HTML.TagSoup.Type
import Text.HTML.TagSoup.Entity
import Text.StringLike


data ParseOptions str = ParseOptions
    {optTagPosition :: Bool -- ^ Should 'TagPosition' values be given before some items (default=False,fast=False)
    ,optTagWarning :: Bool  -- ^ Should 'TagWarning' values be given (default=False,fast=False)
    ,optEntityData :: str -> [Tag str] -- ^ How to lookup an entity
    ,optEntityAttrib :: (str,Bool) -> (str,[Tag str]) -- ^ How to lookup an entity in an attribute (Bool = has ending ';'?)
    ,optTagTextMerge :: Bool -- ^ Require no adjacent 'TagText' values (default=True,fast=False)
    }


parseOptions :: StringLike str => ParseOptions str
parseOptions = ParseOptions False False entityData entityAttrib True
    where
        entityData x = case lookupEntity y of
            Just y -> [TagText $ fromString1 y]
            Nothing -> [TagText $ fromString $ "&" ++ y ++ ";"
                       ,TagWarning $ fromString $ "Unknown entity: " ++ y]
            where y = toString x

        entityAttrib (x,b) = case lookupEntity y of
            Just y -> (fromString1 y, [])
            Nothing -> (fromString $ "&" ++ y ++ [';'|b], [TagWarning $ fromString $ "Unknown entity: " ++ y])
            where y = toString x


parseOptionsFast :: StringLike str => ParseOptions str
parseOptionsFast = parseOptions{optTagTextMerge=False}


parseOptionsRetype :: (StringLike from, StringLike to) => ParseOptions from -> ParseOptions to
parseOptionsRetype (ParseOptions a b c d e) = ParseOptions a b c2 d2 e
    where
        re1 = fromString . toString
        re2 = fromString . toString
        c2 x = map (fmap re1) $ c $ re2 x
        d2 (x,y) = (re1 r, map (fmap re1) s)
            where (r,s) = d (re2 x, y)
<commit_msg>Rename parseOptionsRetype to fmapParseOptions<commit_after>
module Text.HTML.TagSoup.Options_ where

import Text.HTML.TagSoup.Type
import Text.HTML.TagSoup.Entity
import Text.StringLike


data ParseOptions str = ParseOptions
    {optTagPosition :: Bool -- ^ Should 'TagPosition' values be given before some items (default=False,fast=False)
    ,optTagWarning :: Bool  -- ^ Should 'TagWarning' values be given (default=False,fast=False)
    ,optEntityData :: str -> [Tag str] -- ^ How to lookup an entity
    ,optEntityAttrib :: (str,Bool) -> (str,[Tag str]) -- ^ How to lookup an entity in an attribute (Bool = has ending ';'?)
    ,optTagTextMerge :: Bool -- ^ Require no adjacent 'TagText' values (default=True,fast=False)
    }


parseOptions :: StringLike str => ParseOptions str
parseOptions = ParseOptions False False entityData entityAttrib True
    where
        entityData x = case lookupEntity y of
            Just y -> [TagText $ fromString1 y]
            Nothing -> [TagText $ fromString $ "&" ++ y ++ ";"
                       ,TagWarning $ fromString $ "Unknown entity: " ++ y]
            where y = toString x

        entityAttrib (x,b) = case lookupEntity y of
            Just y -> (fromString1 y, [])
            Nothing -> (fromString $ "&" ++ y ++ [';'|b], [TagWarning $ fromString $ "Unknown entity: " ++ y])
            where y = toString x


parseOptionsFast :: StringLike str => ParseOptions str
parseOptionsFast = parseOptions{optTagTextMerge=False}


fmapParseOptions :: (StringLike from, StringLike to) => ParseOptions from -> ParseOptions to
fmapParseOptions (ParseOptions a b c d e) = ParseOptions a b c2 d2 e
    where
        re1 = fromString . toString
        re2 = fromString . toString
        c2 x = map (fmap re1) $ c $ re2 x
        d2 (x,y) = (re1 r, map (fmap re1) s)
            where (r,s) = d (re2 x, y)

{'fraction_non_alphanumeric': 0.11322808927599347, 'fraction_numerical': 0.007076755579749592, 'mean_word_length': 2.828125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2858748', 'n_tokens_mistral': 1093, 'n_tokens_neox': 1084, 'n_words': 369}