<commit_before># encoding: utf-8
require 'csv'
require 'charlock_holmes'
require 'tempfile'
require 'fileutils'
require_relative './job'
require_relative './source_file'

module CartoDB
  module Importer2
    class CsvNormalizer
      LINE_LIMIT            = 1000      # Max line size
      SAMPLE_READ_LIMIT     = 2000000   # Read big enough sample bytes for the encoding sampling
      COMMON_DELIMITERS     = [',', "\t", ' ', ';']
      DEFAULT_DELIMITER     = ','
      DEFAULT_ENCODING      = 'UTF-8'
      ENCODING_CONFIDENCE   = 30 
      ACCEPTABLE_ENCODINGS  = %w{ ISO-8859-1 ISO-8859-2 UTF-8 }

      def self.supported?(extension)
        %w(.csv .tsv .txt).include?(extension)
      end #self.supported?

      def initialize(filepath, job=nil)
        @filepath = filepath
        @job      = job || Job.new
      end #initialize

      def normalize(temporary_filepath)
        temporary_csv = ::CSV.open(temporary_filepath, 'w', col_sep: ',')

        File.open(filepath, 'rb', external_encoding: encoding)
        .each_line(line_delimiter) { |line| 
          row = parsed_line(line)
          next unless row
          temporary_csv << multiple_column(row)
        }

        temporary_csv.close
      rescue ArgumentError
        raise EncodingDetectionError
      end
<commit_msg>CDB-1528 Lowered from 2MB to 500KB<commit_after># encoding: utf-8
require 'csv'
require 'charlock_holmes'
require 'tempfile'
require 'fileutils'
require_relative './job'
require_relative './source_file'

module CartoDB
  module Importer2
    class CsvNormalizer
      LINE_LIMIT            = 1000      # Max line size
      SAMPLE_READ_LIMIT     = 500000   # Read big enough sample bytes for the encoding sampling
      COMMON_DELIMITERS     = [',', "\t", ' ', ';']
      DEFAULT_DELIMITER     = ','
      DEFAULT_ENCODING      = 'UTF-8'
      ENCODING_CONFIDENCE   = 30 
      ACCEPTABLE_ENCODINGS  = %w{ ISO-8859-1 ISO-8859-2 UTF-8 }

      def self.supported?(extension)
        %w(.csv .tsv .txt).include?(extension)
      end #self.supported?

      def initialize(filepath, job=nil)
        @filepath = filepath
        @job      = job || Job.new
      end #initialize

      def normalize(temporary_filepath)
        temporary_csv = ::CSV.open(temporary_filepath, 'w', col_sep: ',')

        File.open(filepath, 'rb', external_encoding: encoding)
        .each_line(line_delimiter) { |line| 
          row = parsed_line(line)
          next unless row
          temporary_csv << multiple_column(row)
        }

        temporary_csv.close
      rescue ArgumentError
        raise EncodingDetectionError
      end

{'fraction_non_alphanumeric': 0.09885496183206106, 'fraction_numerical': 0.023282442748091603, 'mean_word_length': 2.4083224967490247, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5209973', 'n_tokens_mistral': 869, 'n_tokens_neox': 810, 'n_words': 187}