<commit_before>module Patterns
  extend ActiveSupport::Concern

  SPECIAL_CHARACTERS    = ".-_".freeze
  ALLOWED_CHARACTERS    = "[A-Za-z0-9#{Regexp.escape(SPECIAL_CHARACTERS)}]+".freeze
  ROUTE_PATTERN         = /#{ALLOWED_CHARACTERS}/
  LAZY_ROUTE_PATTERN    = /#{ALLOWED_CHARACTERS}?/
  NAME_PATTERN          = /\A#{ALLOWED_CHARACTERS}\Z/
  URL_VALIDATION_REGEXP = %r{\Ahttps?:\/\/([^\s:@]+:[^\s:@]*@)?[A-Za-z\d\-]+(\.[A-Za-z\d\-]+)+\.?(:\d{1,5})?([\/?]\S*)?\z}
  GEM_NAME_BLACKLIST    = %w[
    abbrev
    base64
    benchmark
    cgi
    cgi-session
    complex
    continuation
    coverage
    delegate
    digest
    drb
    e2mmap
    english
    enumerator
    erb
    expect
    fiber
    find
    forwardable
    getoptlong
    install
    io-nonblock
    io-wait
    irb
    jruby
    logger
    matrix
    mkmf
    monitor
    mri
    mruby
    mutex_m
    net-ftp
    net-http
    net-imap
    net-pop
    net-protocol
    net-smtp
    nkf
    observer
    open-uri
    open3
    optparse
    ostruct
    pathname
    prettyprint
    prime
    profile
    profiler
    pstore
    pty
    rational
    rbconfig
    resolv
    rexml
    rinda
    rss
    ruby
    rubygems
    securerandom
    set
    shellwords
    singleton
    socket
    stringio
    strscan
    sync
    syslog
    tempfile
    thread
    thwait
    time
    timeout
    tmpdir
    tsort
    un
    unicode_normalize
    uninstall
    uri
    weakref
    win32ole
    yaml
    ubygems
    sidekiq-pro
  ].freeze
end<commit_msg>Add graphql-pro to blacklist (like sidekiq-pro)<commit_after>module Patterns
  extend ActiveSupport::Concern

  SPECIAL_CHARACTERS    = ".-_".freeze
  ALLOWED_CHARACTERS    = "[A-Za-z0-9#{Regexp.escape(SPECIAL_CHARACTERS)}]+".freeze
  ROUTE_PATTERN         = /#{ALLOWED_CHARACTERS}/
  LAZY_ROUTE_PATTERN    = /#{ALLOWED_CHARACTERS}?/
  NAME_PATTERN          = /\A#{ALLOWED_CHARACTERS}\Z/
  URL_VALIDATION_REGEXP = %r{\Ahttps?:\/\/([^\s:@]+:[^\s:@]*@)?[A-Za-z\d\-]+(\.[A-Za-z\d\-]+)+\.?(:\d{1,5})?([\/?]\S*)?\z}
  GEM_NAME_BLACKLIST    = %w[
    abbrev
    base64
    benchmark
    cgi
    cgi-session
    complex
    continuation
    coverage
    delegate
    digest
    drb
    e2mmap
    english
    enumerator
    erb
    expect
    fiber
    find
    forwardable
    getoptlong
    install
    io-nonblock
    io-wait
    irb
    jruby
    logger
    matrix
    mkmf
    monitor
    mri
    mruby
    mutex_m
    net-ftp
    net-http
    net-imap
    net-pop
    net-protocol
    net-smtp
    nkf
    observer
    open-uri
    open3
    optparse
    ostruct
    pathname
    prettyprint
    prime
    profile
    profiler
    pstore
    pty
    rational
    rbconfig
    resolv
    rexml
    rinda
    rss
    ruby
    rubygems
    securerandom
    set
    shellwords
    singleton
    socket
    stringio
    strscan
    sync
    syslog
    tempfile
    thread
    thwait
    time
    timeout
    tmpdir
    tsort
    un
    unicode_normalize
    uninstall
    uri
    weakref
    win32ole
    yaml
    ubygems
    sidekiq-pro
    graphql-pro
  ].freeze
end
{'fraction_non_alphanumeric': 0.0907612231620039, 'fraction_numerical': 0.006506180871828237, 'mean_word_length': 1.7702702702702702, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3342302', 'n_tokens_mistral': 1220, 'n_tokens_neox': 1182, 'n_words': 213}