<commit_before># encoding: utf-8
require 'csv'
require 'roo'
require_relative './job'

module CartoDB
  module Importer2
    class Xls2Csv
      def self.supported?(extension)
        extension == '.xls'
      end #self.supported?

      def initialize(filepath, job=nil)
        @filepath = filepath
        @job      = job || Job.new
      end #initialize

      def run
        job.log 'Converting XLS to CSV'
        spreadsheet = Roo::Excel.new(filepath, nil, :ignore)
        spreadsheet.to_csv(converted_filepath)
        self
      end #run

      def converted_filepath
        File.join(
          File.dirname(filepath),
          File.basename(filepath, File.extname(filepath))
        ) + '.csv'
      end #converted_filepath

      private

      attr_reader :filepath, :job
    end #Xls2Csv
  end # Importer2
end # CartoDB

<commit_msg>[importer] Normalize XLS files after converting to CSV<commit_after># encoding: utf-8
require 'csv'
require 'roo'
require_relative './job'
require_relative './csv_normalizer'

module CartoDB
  module Importer2
    class Xls2Csv
      def self.supported?(extension)
        extension == '.xls'
      end #self.supported?

      def initialize(filepath, job=nil)
        @filepath = filepath
        @job      = job || Job.new
      end #initialize

      def run
        job.log 'Converting XLS to CSV'
        spreadsheet = Roo::Spreadsheet.open(filepath)
        spreadsheet.to_csv(converted_filepath)
        CsvNormalizer.new(converted_filepath, job).run
        self
      end #run

      def converted_filepath
        File.join(
          File.dirname(filepath),
          File.basename(filepath, File.extname(filepath))
        ) + '.csv'
      end #converted_filepath

      private

      attr_reader :filepath, :job
    end #Xls2Csv
  end # Importer2
end # CartoDB


{'fraction_non_alphanumeric': 0.09085933223864258, 'fraction_numerical': 0.005473453749315818, 'mean_word_length': 2.3115942028985508, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6070349', 'n_tokens_mistral': 589, 'n_tokens_neox': 571, 'n_words': 139}