<commit_before>require 'zip'

module FileExtractor

  class ZipExtractor

    def initialize file
      @file = file
    end

    def file_list
      list = []

      begin
        list = Zip::File.open(@file) do |zip_file|
          zip_file.map(&:name)
        end

      rescue Zip::Error => e
        Rails.logger.error("[zip error] #{e.message}")
      end

      list
    end

    def get_data
      {}
    end

  end

end<commit_msg>Correct UTF-8 bug on strings that come from .zip files<commit_after>require 'zip'

module FileExtractor

  class ZipExtractor

    def initialize file
      @file = file
    end

    def file_list
      list = []

      begin
        list = Zip::File.open(@file) do |zip_file|
          # This is a source of external strings which are shown in the interface
          # so I'll just convert it to UTF-8 here to prevent problems
          zip_file.map do |file|
            file.name.force_encoding("UTF-8")
          end
        end

      rescue Zip::Error => e
        Rails.logger.error("[zip error] #{e.message}")
      end

      list
    end

    def get_data
      {}
    end

  end

end
{'fraction_non_alphanumeric': 0.08370044052863436, 'fraction_numerical': 0.0026431718061674008, 'mean_word_length': 1.7843137254901962, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4537059', 'n_tokens_mistral': 361, 'n_tokens_neox': 340, 'n_words': 108}