<commit_before>module Dictionary
  require 'dbf'

  class Record
    attr_reader :records, :name

    def initialize(config)
      Dictionary.logger.debug "Starting create Records for #{config}"
      @records = Hash.new
      filename = nil

      dict_name = config['name']

      paths = Pathname.new(File.expand_path('../.') + '/tmp/' + dict_name).children.select { |c| c.directory? }.collect { |p| p.to_s }

      paths.each do |path|
        @records[File.basename path.gsub('_', '.')] = Hash.new

        config['dictionaries'].each do |dictionary_key, dictionary_value|
          @records[File.basename path.gsub('_', '.')][dictionary_key.to_s.downcase.pluralize.to_sym] = Hash.new

          if dictionary_value['format'] == 'txt'
            Dictionary.logger.debug " Processing text file for #{config}"
            if File.exist?(path + '/' + dictionary_key.to_s + '.txt')
              filename = File.open(path.to_s + '/' + dictionary_key.to_s + '.txt')
            else
              Dictionary.logger.error "File #{path.to_s + '/' + dictionary_key.to_s}.txt doesn\'t exist"
              exit
            end

            delimiter = dictionary_value['delimiter'].to_i.chr(dictionary_value['encoding'].to_s)

            index = 0

            filename.each do |line|
              index += 1
              line = line.to_s.encode('UTF-8', dictionary_value['encoding'].to_s).delete("\r\n")
              @records[File.basename path.gsub('_', '.')][dictionary_key.to_s.downcase.pluralize.to_sym][index] = is_record(line, delimiter.encode('UTF-8'), dictionary_value['fields'])
            end

          elsif dictionary_value['format'] == 'dbf'
            Dictionary.logger.debug " Processing DBase file for #{config}"
            if File.exist?(path + '/' + dictionary_key.to_s + '.dbf')
              Dictionary.logger.debug "   Filename #{path.to_s}/#{dictionary_key.to_s}.dbf"
              filename = DBF::Table.new("#{path.to_s}/#{dictionary_key.to_s}.dbf", nil, dictionary_value['encoding'].to_s)
            else
              Dictionary.logger.error "File #{path.to_s + '/' + dictionary_key.to_s}.dbf doesn\'t exist"
              exit
            end

            index = 0

            filename.each do |record|
              index += 1
              record.attributes.each do |key, value|
                @records[File.basename path.gsub('_', '.')][dictionary_key.to_s.downcase.pluralize.to_sym][index] = Hash.new if @records[File.basename path.gsub('_', '.')][dictionary_key.to_s.downcase.pluralize.to_sym][index].nil?
                @records[File.basename path.gsub('_', '.')][dictionary_key.to_s.downcase.pluralize.to_sym][index][key.to_s.downcase.to_sym.to_s] = value.to_s.encode( 'UTF-8', dictionary_value['encoding'])
              end
            end
          end
        end
      end
    end<commit_msg>Added some magic for loading file starting from line different than firs one<commit_after>module Dictionary
  require 'dbf'

  class Record
    attr_reader :records, :name

    def initialize(config)
      Dictionary.logger.debug "Starting create Records for #{config}"
      @records = Hash.new
      filename = nil

      dict_name = config['name']

      paths = Pathname.new(File.expand_path('../.') + '/tmp/' + dict_name).children.select { |c| c.directory? }.collect { |p| p.to_s }

      paths.each do |path|
        @records[File.basename path.gsub('_', '.')] = Hash.new

        config['dictionaries'].each do |dictionary_key, dictionary_value|
          @records[File.basename path.gsub('_', '.')][dictionary_key.to_s.downcase.pluralize.to_sym] = Hash.new

          config['dictionaries']['file'].has_key?('startfrom') ? start = config['dictionaries']['file']['startfrom'] : start = 1

          if dictionary_value['format'] == 'txt'
            Dictionary.logger.debug " Processing text file for #{config}"
            if File.exist?(path + '/' + dictionary_key.to_s + '.txt')
              filename = File.open(path.to_s + '/' + dictionary_key.to_s + '.txt')
            else
              Dictionary.logger.error "File #{path.to_s + '/' + dictionary_key.to_s}.txt doesn\'t exist"
              exit
            end

            delimiter = dictionary_value['delimiter'].to_i.chr(dictionary_value['encoding'].to_s)

            index = 0

            filename.each do |line|
              index += 1
              if index >= start
                line = line.to_s.encode('UTF-8', dictionary_value['encoding'].to_s).delete("\r\n")
                @records[File.basename path.gsub('_', '.')][dictionary_key.to_s.downcase.pluralize.to_sym][index] = is_record(line, delimiter.encode('UTF-8'), dictionary_value['fields'])
              end
            end

          elsif dictionary_value['format'] == 'dbf'
            Dictionary.logger.debug " Processing DBase file for #{config}"
            if File.exist?(path + '/' + dictionary_key.to_s + '.dbf')
              Dictionary.logger.debug "   Filename #{path.to_s}/#{dictionary_key.to_s}.dbf"
              filename = DBF::Table.new("#{path.to_s}/#{dictionary_key.to_s}.dbf", nil, dictionary_value['encoding'].to_s)
            else
              Dictionary.logger.error "File #{path.to_s + '/' + dictionary_key.to_s}.dbf doesn\'t exist"
              exit
            end

            index = 0

            filename.each do |record|
              index += 1
              record.attributes.each do |key, value|
                @records[File.basename path.gsub('_', '.')][dictionary_key.to_s.downcase.pluralize.to_sym][index] = Hash.new if @records[File.basename path.gsub('_', '.')][dictionary_key.to_s.downcase.pluralize.to_sym][index].nil?
                @records[File.basename path.gsub('_', '.')][dictionary_key.to_s.downcase.pluralize.to_sym][index][key.to_s.downcase.to_sym.to_s] = value.to_s.encode( 'UTF-8', dictionary_value['encoding'])
              end
            end
          else
            Dictionary.logger.fatal "Unknown format"
          end
        end
      end
    end
{'fraction_non_alphanumeric': 0.14807403701850924, 'fraction_numerical': 0.0025012506253126563, 'mean_word_length': 2.3229916897506926, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2585964', 'n_tokens_mistral': 1870, 'n_tokens_neox': 1818, 'n_words': 326}