<commit_before>GEM
  remote: https://rubygems.org/
  specs:
    activesupport (6.1.4.1)
      concurrent-ruby (~> 1.0, >= 1.0.2)
      i18n (>= 1.6, < 2)
      minitest (>= 5.1)
      tzinfo (~> 2.0)
      zeitwerk (~> 2.3)
    addressable (2.8.0)
      public_suffix (>= 2.0.2, < 5.0)
    colorator (1.1.0)
    concurrent-ruby (1.1.9)
    em-websocket (0.5.3)
      eventmachine (>= 0.12.9)
      http_parser.rb (~> 0)
    ethon (0.15.0)
      ffi (>= 1.15.0)
    eventmachine (1.2.7)
    ffi (1.15.4)
    forwardable-extended (2.6.0)
    gemoji (3.0.1)
    html-pipeline (2.14.0)
      activesupport (>= 2)
      nokogiri (>= 1.4)
    html-proofer (3.19.2)
      addressable (~> 2.3)
      mercenary (~> 0.3)
      nokogumbo (~> 2.0)
      parallel (~> 1.3)
      rainbow (~> 3.0)
      typhoeus (~> 1.3)
      yell (~> 2.0)
    http_parser.rb (0.8.0)
    i18n (1.8.11)
      concurrent-ruby (~> 1.0)
    jekyll (4.2.1)
      addressable (~> 2.4)
      colorator (~> 1.0)
      em-websocket (~> 0.5)
      i18n (~> 1.0)
      jekyll-sass-converter (~> 2.0)
      jekyll-watch (~> 2.0)
      kramdown (~> 2.3)
      kramdown-parser-gfm (~> 1.0)
      liquid (~> 4.0)
      mercenary (~> 0.4.0)
      pathutil (~> 0.9)
      rouge (~> 3.0)
      safe_yaml (~> 1.0)
      terminal-table (~> 2.0)
    jekyll-feed (0.15.1)
      jekyll (>= 3.7, < 5.0)
    jekyll-paginate (1.1.0)
    jekyll-sass-converter (2.1.0)
      sassc (> 2.0.1, < 3.0)
    jekyll-seo-tag (2.7.1)
      jekyll (>= 3.8, < 5.0)
    jekyll-watch (2.2.1)
      listen (~> 3.0)
    jemoji (0.12.0)
      gemoji (~> 3.0)
      html-pipeline (~> 2.2)
      jekyll (>= 3.0, < 5.0)
    kramdown (2.3.1)
      rexml
    kramdown-parser-gfm (1.1.0)
      kramdown (~> 2.0)
    liquid (4.0.3)
    listen (3.7.0)
      rb-fsevent (~> 0.10, >= 0.10.3)
      rb-inotify (~> 0.9, >= 0.9.10)
    mercenary (0.4.0)
    mini_portile2 (2.6.1)
    minima (2.5.1)
      jekyll (>= 3.5, < 5.0)
      jekyll-feed (~> 0.9)
      jekyll-seo-tag (~> 2.1)
    minitest (5.14.4)
    nokogiri (1.12.5)
      mini_portile2 (~> 2.6.1)
      racc (~> 1.4)
    nokogiri (1.12.5-x86_64-linux)
      racc (~> 1.4)
    nokogumbo (2.0.5)
      nokogiri (~> 1.8, >= 1.8.4)
    parallel (1.21.0)
    pathutil (0.16.2)
      forwardable-extended (~> 2.6)
    public_suffix (4.0.6)
    racc (1.6.0)
    rainbow (3.0.0)
    rb-fsevent (0.11.0)
    rb-inotify (0.10.1)
      ffi (~> 1.0)
    rexml (3.2.5)
    rouge (3.26.1)
    safe_yaml (1.0.5)
    sassc (2.4.0)
      ffi (~> 1.9)
    terminal-table (2.0.0)
      unicode-display_width (~> 1.1, >= 1.1.1)
    typhoeus (1.4.0)
      ethon (>= 0.9.0)
    tzinfo (2.0.4)
      concurrent-ruby (~> 1.0)
    unicode-display_width (1.8.0)
    wdm (0.1.1)
    yell (2.2.2)
    zeitwerk (2.5.1)
<commit_msg>update gemfile<commit_after>GEM
  remote: https://rubygems.org/
  specs:
    activesupport (7.0.0)
      concurrent-ruby (~> 1.0, >= 1.0.2)
      i18n (>= 1.6, < 2)
      minitest (>= 5.1)
      tzinfo (~> 2.0)
    addressable (2.8.0)
      public_suffix (>= 2.0.2, < 5.0)
    colorator (1.1.0)
    concurrent-ruby (1.1.9)
    em-websocket (0.5.3)
      eventmachine (>= 0.12.9)
      http_parser.rb (~> 0)
    ethon (0.15.0)
      ffi (>= 1.15.0)
    eventmachine (1.2.7)
    ffi (1.15.4)
    forwardable-extended (2.6.0)
    gemoji (3.0.1)
    html-pipeline (2.14.0)
      activesupport (>= 2)
      nokogiri (>= 1.4)
    html-proofer (3.19.3)
      addressable (~> 2.3)
      mercenary (~> 0.3)
      nokogiri (~> 1.12)
      parallel (~> 1.3)
      rainbow (~> 3.0)
      typhoeus (~> 1.3)
      yell (~> 2.0)
    http_parser.rb (0.8.0)
    i18n (1.8.11)
      concurrent-ruby (~> 1.0)
    jekyll (4.2.1)
      addressable (~> 2.4)
      colorator (~> 1.0)
      em-websocket (~> 0.5)
      i18n (~> 1.0)
      jekyll-sass-converter (~> 2.0)
      jekyll-watch (~> 2.0)
      kramdown (~> 2.3)
      kramdown-parser-gfm (~> 1.0)
      liquid (~> 4.0)
      mercenary (~> 0.4.0)
      pathutil (~> 0.9)
      rouge (~> 3.0)
      safe_yaml (~> 1.0)
      terminal-table (~> 2.0)
    jekyll-feed (0.15.1)
      jekyll (>= 3.7, < 5.0)
    jekyll-paginate (1.1.0)
    jekyll-sass-converter (2.1.0)
      sassc (> 2.0.1, < 3.0)
    jekyll-seo-tag (2.7.1)
      jekyll (>= 3.8, < 5.0)
    jekyll-watch (2.2.1)
      listen (~> 3.0)
    jemoji (0.12.0)
      gemoji (~> 3.0)
      html-pipeline (~> 2.2)
      jekyll (>= 3.0, < 5.0)
    kramdown (2.3.1)
      rexml
    kramdown-parser-gfm (1.1.0)
      kramdown (~> 2.0)
    liquid (4.0.3)
    listen (3.7.0)
      rb-fsevent (~> 0.10, >= 0.10.3)
      rb-inotify (~> 0.9, >= 0.9.10)
    mercenary (0.4.0)
    mini_portile2 (2.6.1)
    minima (2.5.1)
      jekyll (>= 3.5, < 5.0)
      jekyll-feed (~> 0.9)
      jekyll-seo-tag (~> 2.1)
    minitest (5.15.0)
    nokogiri (1.12.5)
      mini_portile2 (~> 2.6.1)
      racc (~> 1.4)
    nokogiri (1.12.5-x86_64-linux)
      racc (~> 1.4)
    parallel (1.21.0)
    pathutil (0.16.2)
      forwardable-extended (~> 2.6)
    public_suffix (4.0.6)
    racc (1.6.0)
    rainbow (3.0.0)
    rb-fsevent (0.11.0)
    rb-inotify (0.10.1)
      ffi (~> 1.0)
    rexml (3.2.5)
    rouge (3.27.0)
    safe_yaml (1.0.5)
    sassc (2.4.0)
      ffi (~> 1.9)
    terminal-table (2.0.0)
      unicode-display_width (~> 1.1, >= 1.1.1)
    typhoeus (1.4.0)
      ethon (>= 0.9.0)
    tzinfo (2.0.4)
      concurrent-ruby (~> 1.0)
    unicode-display_width (1.8.0)
    wdm (0.1.1)
    yell (2.2.2)

{'fraction_non_alphanumeric': 0.20828749541620828, 'fraction_numerical': 0.1178951228456179, 'mean_word_length': 1.6087996174079389, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 122, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3542404', 'n_tokens_mistral': 2990, 'n_tokens_neox': 2747, 'n_words': 450}