<commit_before>require "yaml"
require "set"
require "bad_word_detector/state"
require "bad_word_detector/prefix_tree"
require "bad_word_detector/rule"
require "bad_word_detector/version"
require "bad_word_detector/whitelist"
require "bad_word_detector/bad_word"

class BadWordDetector

  # Create new badword checker
  # 
  # @param rules [Hash<String,Array<Hash<String, any>>>] Hash where values are arrays 
  #   of Hash<['symbol', 'weight'], any> where weight is optional
  # 
  # @param library [Array<String>] Array of bad words to find
  # 
  # @param whitelist [Array<String>] Array of words that is acceptable. Used in false-positive check
  # 
  def initialize(rules = nil, library = nil,  whitelist = nil)
    confdir = File.expand_path(File.dirname(__FILE__) + "/conf")
    rules ||= YAML.load_file("#{confdir}/rules.yaml")
    library ||= YAML.load_file("#{confdir}/library.yaml")

    @rule_sets = rules.select do |key, _|
      key.to_s.length == 1
    end.hmap do |key, rule|
      key = key.to_s
      rule = rule.map do |item|
        Rule.new(key, item['symbol'], item['weight'])
      end
      rule << Rule.new(key, key, 3)
      [key, rule]
    end

    @string_sets = rules.select do |key, _|
      key.to_s.length > 1
    end.hmap do |key, rule|
      key = key.to_s
      rule = rule.map do |item|
        Rule.new(key, item['symbol'], item['weight'])
      end
      [key, rule]
    end

    @library = PrefixTree.new library
    @whitelist = Whitelist.new whitelist || YAML.load_file("#{confdir}/whitelist.yaml")
  end

  #
  # Searches string for some word in library
  # 
  # @param text [String] String to search
  # 
  # @param return_white [Boolean] Flag to indicate if search should return whitelist word
  # 
  # @return [BadWord, nil] BadWord object, containing information about found word and it's position in text
  # 
  def find(text, return_white = false)
    downcased = text.downcase
    length = text.length
    index = 0
    while index < length
      found = find_part(downcased, index)
      if found 
        word = BadWord.new(
          found[:word], 
          text, 
          index, 
          found[:length], 
          @whitelist)
        if not word.white? or return_white
          return word
        end
      end
      index += 1
    end<commit_msg>Update bad_word_detector.rb<commit_after>require "yaml"
require "set"
require "bad_word_detector/state"
require "bad_word_detector/prefix_tree"
require "bad_word_detector/rule"
require "bad_word_detector/version"
require "bad_word_detector/whitelist"
require "bad_word_detector/bad_word"

class BadWordDetector

  # Create new badword checker
  # 
  # @param rules [Hash<String,Array<Hash<String, any>>>] Hash where values are arrays 
  #   of Hash<['symbol', 'weight'], any> where weight is optional
  # 
  # @param library [Array<String>] Array of bad words to find
  # 
  # @param whitelist [Array<String>] Array of words that is acceptable. Used in false-positive check
  # 
  def initialize(rules = nil, library = nil,  whitelist = nil)
    confdir = File.expand_path(File.dirname(__FILE__) + "/conf")
    rules ||= YAML.load_file("#{confdir}/rules.yaml")
    library ||= YAML.load_file("#{confdir}/library.yaml")

    @rule_sets = rules.select do |key, _|
      key.to_s.length == 1
    end.hmap do |key, rule|
      key = key.to_s
      rule = rule.map do |item|
        Rule.new(key, item['symbol'], item['weight'])
      end
      rule << Rule.new(key, key, 3)
      [key, rule]
    end

    @string_sets = rules.select do |key, _|
      key.to_s.length > 1
    end.hmap do |key, rule|
      key = key.to_s
      rule = rule.map do |item|
        Rule.new(key, item['symbol'], item['weight'])
      end
      [key, rule]
    end

    @library = PrefixTree.new library
    @whitelist = Whitelist.new whitelist || YAML.load_file("#{confdir}/whitelist.yaml")
    true
  end

  #
  # Searches string for some word in library
  # 
  # @param text [String] String to search
  # 
  # @param return_white [Boolean] Flag to indicate if search should return whitelist word
  # 
  # @return [BadWord, nil] BadWord object, containing information about found word and it's position in text
  # 
  def find(text, return_white = false)
    downcased = text.downcase
    length = text.length
    index = 0
    while index < length
      found = find_part(downcased, index)
      if found 
        word = BadWord.new(
          found[:word], 
          text, 
          index, 
          found[:length], 
          @whitelist)
        if not word.white? or return_white
          return word
        end
      end
      index += 1
    end
{'fraction_non_alphanumeric': 0.11278844080224283, 'fraction_numerical': 0.0021565667457407807, 'mean_word_length': 2.740322580645161, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3066767', 'n_tokens_mistral': 1554, 'n_tokens_neox': 1481, 'n_words': 421}