<commit_before>require "echidna/stopwords/version"

module Echidna
  module Stopwords
    CHINESE_DICT_PATH = File.join(File.dirname(__FILE__), '../../dicts/chinese_stopwords.txt')
    ENGLISH_DICT_PATH = File.join(File.dirname(__FILE__), '../../dicts/english_stopwords.txt')

    class <<self
      def load
        load_dict(CHINESE_DICT_PATH)
        load_dict(ENGLISH_DICT_PATH)
      end

      def reject(words)
        # for every word that is not single character and not a username and is not a stopword
        words.select { |word| !single_character?(word) && !username?(word) && !is?(word) }
      end

      # add a stopword (TODO: rename)
      def add(word)
        $redis.sadd key, word
      end

      # remove all the stopwords by removing the set
      def flush
        $redis.del key
      end

      # is this a stopword?
      def is?(word)
        $redis.sismember key, word
      end

      # is this a single character word?
      def single_character?(word)
        word.length == 1
      end

      # tencent usernames start with @
      def username?(word)
        word[0] == '@'
      end

      private
      def key
        "stopwords"
      end

      def load_dict(filename)
        File.open(filename, 'r') do |file|
          file.each_line do |line|
            Stopword.add(line.strip)
          end
        end
      end<commit_msg>no Stopword class now<commit_after>require "echidna/stopwords/version"

module Echidna
  module Stopwords
    CHINESE_DICT_PATH = File.join(File.dirname(__FILE__), '../../dicts/chinese_stopwords.txt')
    ENGLISH_DICT_PATH = File.join(File.dirname(__FILE__), '../../dicts/english_stopwords.txt')

    class <<self
      def load
        load_dict(CHINESE_DICT_PATH)
        load_dict(ENGLISH_DICT_PATH)
      end

      def reject(words)
        # for every word that is not single character and not a username and is not a stopword
        words.select { |word| !single_character?(word) && !username?(word) && !is?(word) }
      end

      # add a stopword (TODO: rename)
      def add(word)
        $redis.sadd key, word
      end

      # remove all the stopwords by removing the set
      def flush
        $redis.del key
      end

      # is this a stopword?
      def is?(word)
        $redis.sismember key, word
      end

      # is this a single character word?
      def single_character?(word)
        word.length == 1
      end

      # tencent usernames start with @
      def username?(word)
        word[0] == '@'
      end

      private
      def key
        "stopwords"
      end

      def load_dict(filename)
        File.open(filename, 'r') do |file|
          file.each_line do |line|
            add(line.strip)
          end
        end
      end
{'fraction_non_alphanumeric': 0.09588042289464091, 'fraction_numerical': 0.0014582573824279985, 'mean_word_length': 1.9923664122137406, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7186933', 'n_tokens_mistral': 870, 'n_tokens_neox': 861, 'n_words': 232}