<commit_before># encoding: utf-8

require 'unicode_utils/each_word'
require 'uri'

module YUITweets; class Bayes
  # Confidence difference. A score for a particular type must be at least this
  # much higher than the score for any other type before that score will be
  # considered a confident classification. This requirement is in addition to
  # the CONFIDENCE_MIN requirement.
  CONFIDENCE_DIFF = 0.4

  # Confidence minimum. A score for a particular type must be at least this high
  # before it will be considered a confident classification. This requirement is
  # in addition to the CONFIDENCE_DIFF requirement.
  CONFIDENCE_MIN = 0.5

  # Regex that matches a complete word string that should be excluded from the
  # corpus.
  #
  # Currently excludes strings that consist entirely of:
  #   - whitespace
  #   - punctuation
  #   - digits
  #   - a single alphanumeric character
  #
  REGEX_EXCLUDE = /^(?:\s+|[[:punct:]]+|\d+|\w)$/

  # Regex that matches a Twitter #hashtag.
  REGEX_HASHTAG = /(?<=^|\W)#(\w{1,32})(?=\W)?/

  # Regex that matches a Twitter @mention.
  REGEX_MENTION = /(?<=^|\W)@(\w{1,16})(?=\W)?/

  # Regex that matches URIs.
  REGEX_URI = URI.regexp(['ftp', 'http', 'https'])

  # Default probability that an unknown word is of a particular classification<commit_msg>Decrease confidence thresholds.<commit_after># encoding: utf-8

require 'unicode_utils/each_word'
require 'uri'

module YUITweets; class Bayes
  # Confidence difference. A score for a particular type must be at least this
  # much higher than the score for any other type before that score will be
  # considered a confident classification. This requirement is in addition to
  # the CONFIDENCE_MIN requirement.
  CONFIDENCE_DIFF = 0.3

  # Confidence minimum. A score for a particular type must be at least this high
  # before it will be considered a confident classification. This requirement is
  # in addition to the CONFIDENCE_DIFF requirement.
  CONFIDENCE_MIN = 0.4

  # Regex that matches a complete word string that should be excluded from the
  # corpus.
  #
  # Currently excludes strings that consist entirely of:
  #   - whitespace
  #   - punctuation
  #   - digits
  #   - a single alphanumeric character
  #
  REGEX_EXCLUDE = /^(?:\s+|[[:punct:]]+|\d+|\w)$/

  # Regex that matches a Twitter #hashtag.
  REGEX_HASHTAG = /(?<=^|\W)#(\w{1,32})(?=\W)?/

  # Regex that matches a Twitter @mention.
  REGEX_MENTION = /(?<=^|\W)@(\w{1,16})(?=\W)?/

  # Regex that matches URIs.
  REGEX_URI = URI.regexp(['ftp', 'http', 'https'])

  # Default probability that an unknown word is of a particular classification
{'fraction_non_alphanumeric': 0.1064885496183206, 'fraction_numerical': 0.008396946564885497, 'mean_word_length': 3.61443661971831, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1643114', 'n_tokens_mistral': 861, 'n_tokens_neox': 810, 'n_words': 303}