<commit_before># -*- encoding : utf-8 -*-
require 'csv'
module Import
  class Relic
    Fields = [ :identification, :group, :number, :materail, :dating_of_obj, :register_number, :street ]
    RowSize = 11

    class << self

      def structurize_relics
        ::Relic.find_each(:conditions => {:number => 0}) do |parent_relic|
          logger.info "searching children (#{parent_relic.id})"
          parent_relic.find_children
        end
      end

      def logger
        @logger ||= Logger.new("#{Rails.root}/log/#{Rails.env}_relics_import.log")
      end

      def find_place row
        Import::AdministrativeDivision.find_or_create *row.first(4)
      end

      def prepare_attributes row
        attributes = {}
        attributes = Hash[Fields.zip(row.slice(4, RowSize-1))]
        attributes[:internal_id] = (Digest::SHA1.new << row.join).to_s
        attributes
      end

      def prepare_source row
        fields = [:voivodeship, :commune, :district, :place] + Fields
        Hash[fields.zip(row)]
      end

      def parse(file_path = nil)
        file_path = "#{Rails.root}/vendor/csvs/all_relics.csv" if file_path.blank?
        stats = {
          :updated => 0,
          :created => 0,
          :failed  => 0
        }
        logger.info "=== Import stated at: #{Time.now} ==="
<commit_msg>fix order in prepare_sorce method<commit_after># -*- encoding : utf-8 -*-
require 'csv'
module Import
  class Relic
    Fields = [ :identification, :group, :number, :materail, :dating_of_obj, :register_number, :street ]
    RowSize = 11

    class << self

      def structurize_relics
        ::Relic.find_each(:conditions => {:number => 0}) do |parent_relic|
          logger.info "searching children (#{parent_relic.id})"
          parent_relic.find_children
        end
      end

      def logger
        @logger ||= Logger.new("#{Rails.root}/log/#{Rails.env}_relics_import.log")
      end

      def find_place row
        Import::AdministrativeDivision.find_or_create *row.first(4)
      end

      def prepare_attributes row
        attributes = {}
        attributes = Hash[Fields.zip(row.slice(4, RowSize-1))]
        attributes[:internal_id] = (Digest::SHA1.new << row.join).to_s
        attributes
      end

      def prepare_source row
        fields = [:voivodeship, :district, :commune, :place] + Fields
        Hash[fields.zip(row)]
      end

      def parse(file_path = nil)
        file_path = "#{Rails.root}/vendor/csvs/all_relics.csv" if file_path.blank?
        stats = {
          :updated => 0,
          :created => 0,
          :failed  => 0
        }
        logger.info "=== Import stated at: #{Time.now} ==="

{'fraction_non_alphanumeric': 0.1294206170052671, 'fraction_numerical': 0.008276899924755455, 'mean_word_length': 2.31960049937578, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5771380', 'n_tokens_mistral': 874, 'n_tokens_neox': 835, 'n_words': 180}