<commit_before>homepage: http://github.com/bgamari/html-parse
changelog-type: ''
hash: 667c9810f76e0a561b84c96c731c1e2b6e727a2bcc5902b4844ec8f6f231edea
test-bench-deps:
  base: -any
  text: -any
  criterion: ! '>=1.1'
  attoparsec: -any
  deepseq: -any
  tagsoup: ! '>=0.13'
maintainer: <EMAIL>
synopsis: A high-performance HTML tokenizer
changelog: ''
basic-deps:
  base: ! '>=4.8 && <4.10'
  text: ! '>=1.2 && <1.3'
  attoparsec: ! '>=0.13 && <0.14'
  deepseq: ! '>=1.4 && <1.5'
all-versions:
- '0.1.0.0'
- '0.2.0.0'
author: <NAME>
latest: '0.2.0.0'
description-type: haddock
description: ! 'This package provides a fast and reasonably robust HTML5 tokenizer
  built

  upon the @attoparsec@ library. The parsing strategy is based upon the HTML5

  parsing specification with few deviations.


  The package targets similar use-cases to the venerable @tagsoup@ library,

  but is significantly more efficient, achieving parsing speeds of over 50

  megabytes per second on modern hardware with and typical web documents.'
license-name: BSD3<commit_msg>Update from Hackage at 2017-08-10T04:02:02Z<commit_after>homepage: http://github.com/bgamari/html-parse
changelog-type: ''
hash: af4ae2d809e14721f5e620d8755daba3c65a2db089669b60754547cad2726b9d
test-bench-deps:
  base: -any
  hspec: -any
  text: -any
  criterion: ! '>=1.1'
  containers: -any
  quickcheck-instances: -any
  html-parse: -any
  attoparsec: -any
  deepseq: -any
  QuickCheck: -any
  hspec-discover: -any
  string-conversions: -any
  tagsoup: ! '>=0.13'
maintainer: <EMAIL>
synopsis: A high-performance HTML tokenizer
changelog: ''
basic-deps:
  base: ! '>=4.7 && <4.11'
  text: ! '>=1.2 && <1.3'
  containers: ! '>=0.5 && <0.6'
  attoparsec: ! '>=0.13 && <0.14'
  deepseq: ! '>=1.4 && <1.5'
all-versions:
- '0.1.0.0'
- '0.2.0.0'
- '0.2.0.1'
author: <NAME>
latest: '0.2.0.1'
description-type: haddock
description: ! 'This package provides a fast and reasonably robust HTML5 tokenizer
  built

  upon the @attoparsec@ library. The parsing strategy is based upon the HTML5

  parsing specification with few deviations.


  The package targets similar use-cases to the venerable @tagsoup@ library,

  but is significantly more efficient, achieving parsing speeds of over 50

  megabytes per second on modern hardware with and typical web documents.


  For instance,


  >>> parseTokens "<div><h1 class=widget>Hello World</h1><br/>"

  [TagOpen "div" [],TagOpen "h1" [Attr "class" "widget"],

  ContentText "Hello World",TagClose "h1",TagSelfClose "br" []]'
license-name: BSD3
{'fraction_non_alphanumeric': 0.13937007874015747, 'fraction_numerical': 0.07440944881889763, 'mean_word_length': 3.9921414538310414, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 7, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '940076', 'n_tokens_mistral': 1058, 'n_tokens_neox': 960, 'n_words': 261}