<commit_before>! Copyright (C) 2005, 2006 <NAME>
! See http://factorcode.org/license.txt for BSD license.
USING: namespaces make kernel assocs sequences fry
io.files io.encodings.binary xml.state ;
IN: xml.entities

CONSTANT: entities-out
    H{
        { CHAR: < "&lt;"   }
        { CHAR: > "&gt;"   }
        { CHAR: & "&amp;"  }
    }

CONSTANT: quoted-entities-out
    H{
        { CHAR: & "&amp;"  }
        { CHAR: ' "&apos;" }
        { CHAR: \" "&quot;" }
        { CHAR: < "&lt;"   }
    }

: escape-string-by ( str table -- escaped )
    ! Convert <, >, &, ' and " to HTML entities.
    [ '[ dup _ at [ % ] [ , ] ?if ] each ] "" make ;

: escape-string ( str -- newstr )
    entities-out escape-string-by ;

: escape-quoted-string ( str -- newstr )
    quoted-entities-out escape-string-by ;

CONSTANT: entities
    H{
        { "lt"    CHAR: <  }
        { "gt"    CHAR: >  }
        { "amp"   CHAR: &  }
        { "apos"  CHAR: '  }
        { "quot"  CHAR: \"  }
    }

: with-entities ( entities quot -- )
    [ swap extra-entities set call ] with-scope ; inline<commit_msg>xml.entities: use with-variable in with-entities.<commit_after>! Copyright (C) 2005, 2006 <NAME>
! See http://factorcode.org/license.txt for BSD license.
USING: namespaces make kernel assocs sequences fry
io.files io.encodings.binary xml.state ;
IN: xml.entities

CONSTANT: entities-out
    H{
        { CHAR: < "&lt;"   }
        { CHAR: > "&gt;"   }
        { CHAR: & "&amp;"  }
    }

CONSTANT: quoted-entities-out
    H{
        { CHAR: & "&amp;"  }
        { CHAR: ' "&apos;" }
        { CHAR: \" "&quot;" }
        { CHAR: < "&lt;"   }
    }

: escape-string-by ( str table -- escaped )
    ! Convert <, >, &, ' and " to HTML entities.
    [ '[ dup _ at [ % ] [ , ] ?if ] each ] "" make ;

: escape-string ( str -- newstr )
    entities-out escape-string-by ;

: escape-quoted-string ( str -- newstr )
    quoted-entities-out escape-string-by ;

CONSTANT: entities
    H{
        { "lt"    CHAR: <  }
        { "gt"    CHAR: >  }
        { "amp"   CHAR: &  }
        { "apos"  CHAR: '  }
        { "quot"  CHAR: \"  }
    }

: with-entities ( entities quot -- )
    [ extra-entities ] dip with-variable ; inline
{'fraction_non_alphanumeric': 0.1746031746031746, 'fraction_numerical': 0.0072562358276643995, 'mean_word_length': 1.8836601307189542, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 5}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2867641', 'n_tokens_mistral': 824, 'n_tokens_neox': 799, 'n_words': 179}