<commit_before>require 'xml'
class ContactsImportWorker
  include SidekiqStatus::Worker
  sidekiq_options retry: false

  def perform(options)
    account_id = options["account_id"]
    Account.current_id = account_id
    current_account = Account.find(account_id)
    raise "Un autre import est déjà en cours. Merci d'attendre qu'il soit terminé" if current_account.importing_now
    current_account.importing_now = true
    current_account.save!
    user = User.find(options["user_id"])
    filename = options["filename"]
    uploader = ContactsImportUploader.new(account_id.to_s)
    uploader.retrieve_from_store!(options["filename"])
    uploader.cache_stored_file!
    at(10,"Lecture du fichier")
    imported_at = options["imported_at"]
    test_mode = options["test_mode"]
    current_account.destroy_test_contacts
    current_account.test_imported_at = test_mode ? imported_at : nil
    current_account.last_import_at = imported_at unless test_mode
    current_account.importing_now = true
    current_account.save!
    
    case File.extname(filename)
    when ".xml"
      import_xml_file(uploader.file, imported_at, options)
    else
      log_message, error = import_spreadsheet_file(uploader.file, imported_at, options)
    end
    
    if error
      self.payload = { invalid_file: true, message: log_message }
      return
      
    else
      imported_contacts = Contact.where(imported_at: imported_at)
      nb_duplicates = imported_contacts.where("duplicate_id IS NOT NULL").count
      nb_imported_contacts = imported_contacts.count
      self.payload = { nb_imported_contacts: nb_imported_contacts, nb_duplicates: nb_duplicates, imported_at: imported_at, message: log_message }
      UserMailer.contacts_import_email(user, { account: current_account, filename: filename, imported_at: imported_at }).deliver unless test_mode
    end
  ensure
    if current_account
      current_account.importing_now = false
      current_account.save
    end
  end
  

  def import_xml_file(file, imported_at, options)
    ActiveRecord::Base.transaction do
      File.open(file.path) do |io|
        self.total = io.size
        at(20, "Lecture du fichier ...")
        xml_reader = XML::Reader.io(io)
        xml_reader.read
        raise "No valid Merci Edgar File" unless xml_reader.name == "merciedgar"
      
        while xml_reader.read do
          case xml_reader.name
            when "show-buyer", "person", "structure"
              xml_node = xml_reader.expand.to_s
              attributes = Hash.from_xml(xml_node).fetch(xml_reader.name.underscore)
              instance = Object.const_get(xml_reader.name.underscore.camelize).from_merciedgar_hash(attributes, imported_at, options[:custom_tags])
              instance.save
              at(xml_reader.byte_consumed, "Ajout de la fiche #{instance.name}")                
          end
          xml_reader.next
        end
      end

      File.open(file.path) do |io|
        self.total = io.size
        at(20, "Lecture du fichier ...")
        xml_reader = XML::Reader.io(io)
        xml_reader.read
        raise "No valid Merci Edgar File" unless xml_reader.name == "merciedgar"
      
        while xml_reader.read do
          case xml_reader.name
            when "venue", "festival"
              xml_node = xml_reader.expand.to_s
              attributes = Hash.from_xml(xml_node).fetch(xml_reader.name.underscore)
              instance = Object.const_get(xml_reader.name.underscore.camelize).from_merciedgar_hash(attributes, imported_at, options[:custom_tags])
              instance.save
              at(xml_reader.byte_consumed, "Ajout de la fiche #{instance.name}")
                       
                
          end
          xml_reader.next 
        end
      end
    end
    
  end
  
  def import_spreadsheet_file(file, imported_at, options)
    ActiveRecord::Base.transaction do
      spreadsheet = SpreadsheetFile.new(file.path, options["contact_kind"])
      if spreadsheet.invalid?
        log_message = spreadsheet.errors.messages.values.flatten.join(' / ')
        return [ log_message, :invalid_file ]
      end
    
      log_message = imported_at
    
      imported_index = 0
      test_mode = options["test_mode"]
      nb_lines = spreadsheet.nb_lines
    
      total = test_mode && 20 < nb_lines ? 20 : nb_lines
      log_message = ""
      total_chunks = SmarterCSV.process(spreadsheet.csv_path, chunk_size: 100, file_encoding: spreadsheet.encoding, convert_values_to_numeric: {only: [:places_assises, :places_debout, :places_mixte]}) do |chunk|
        chunk.each do |row|
          imported_index += 1
          test_mode = options["test_mode"]
          return if imported_index > 20 && test_mode
          row[:imported_at] = imported_at
          row[:first_name_last_name_order] = options["first_name_last_name_order"]<commit_msg>col separator is ; for csv import<commit_after>require 'xml'
class ContactsImportWorker
  include SidekiqStatus::Worker
  sidekiq_options retry: false

  def perform(options)
    account_id = options["account_id"]
    Account.current_id = account_id
    current_account = Account.find(account_id)
    raise "Un autre import est déjà en cours. Merci d'attendre qu'il soit terminé" if current_account.importing_now
    current_account.importing_now = true
    current_account.save!
    user = User.find(options["user_id"])
    filename = options["filename"]
    uploader = ContactsImportUploader.new(account_id.to_s)
    uploader.retrieve_from_store!(options["filename"])
    uploader.cache_stored_file!
    at(10,"Lecture du fichier")
    imported_at = options["imported_at"]
    test_mode = options["test_mode"]
    current_account.destroy_test_contacts
    current_account.test_imported_at = test_mode ? imported_at : nil
    current_account.last_import_at = imported_at unless test_mode
    current_account.importing_now = true
    current_account.save!
    
    case File.extname(filename)
    when ".xml"
      import_xml_file(uploader.file, imported_at, options)
    else
      log_message, error = import_spreadsheet_file(uploader.file, imported_at, options)
    end
    
    if error
      self.payload = { invalid_file: true, message: log_message }
      return
      
    else
      imported_contacts = Contact.where(imported_at: imported_at)
      nb_duplicates = imported_contacts.where("duplicate_id IS NOT NULL").count
      nb_imported_contacts = imported_contacts.count
      self.payload = { nb_imported_contacts: nb_imported_contacts, nb_duplicates: nb_duplicates, imported_at: imported_at, message: log_message }
      UserMailer.contacts_import_email(user, { account: current_account, filename: filename, imported_at: imported_at }).deliver unless test_mode
    end
  ensure
    if current_account
      current_account.importing_now = false
      current_account.save
    end
  end
  

  def import_xml_file(file, imported_at, options)
    ActiveRecord::Base.transaction do
      File.open(file.path) do |io|
        self.total = io.size
        at(20, "Lecture du fichier ...")
        xml_reader = XML::Reader.io(io)
        xml_reader.read
        raise "No valid Merci Edgar File" unless xml_reader.name == "merciedgar"
      
        while xml_reader.read do
          case xml_reader.name
            when "show-buyer", "person", "structure"
              xml_node = xml_reader.expand.to_s
              attributes = Hash.from_xml(xml_node).fetch(xml_reader.name.underscore)
              instance = Object.const_get(xml_reader.name.underscore.camelize).from_merciedgar_hash(attributes, imported_at, options[:custom_tags])
              instance.save
              at(xml_reader.byte_consumed, "Ajout de la fiche #{instance.name}")                
          end
          xml_reader.next
        end
      end

      File.open(file.path) do |io|
        self.total = io.size
        at(20, "Lecture du fichier ...")
        xml_reader = XML::Reader.io(io)
        xml_reader.read
        raise "No valid Merci Edgar File" unless xml_reader.name == "merciedgar"
      
        while xml_reader.read do
          case xml_reader.name
            when "venue", "festival"
              xml_node = xml_reader.expand.to_s
              attributes = Hash.from_xml(xml_node).fetch(xml_reader.name.underscore)
              instance = Object.const_get(xml_reader.name.underscore.camelize).from_merciedgar_hash(attributes, imported_at, options[:custom_tags])
              instance.save
              at(xml_reader.byte_consumed, "Ajout de la fiche #{instance.name}")
                       
                
          end
          xml_reader.next 
        end
      end
    end
    
  end
  
  def import_spreadsheet_file(file, imported_at, options)
    ActiveRecord::Base.transaction do
      spreadsheet = SpreadsheetFile.new(file.path, options["contact_kind"])
      if spreadsheet.invalid?
        log_message = spreadsheet.errors.messages.values.flatten.join(' / ')
        return [ log_message, :invalid_file ]
      end
    
      log_message = imported_at
    
      imported_index = 0
      test_mode = options["test_mode"]
      nb_lines = spreadsheet.nb_lines
    
      total = test_mode && 20 < nb_lines ? 20 : nb_lines
      log_message = ""
      total_chunks = SmarterCSV.process(spreadsheet.csv_path, col_sep: ';', chunk_size: 100, file_encoding: spreadsheet.encoding, convert_values_to_numeric: {only: [:places_assises, :places_debout, :places_mixte]}) do |chunk|
        chunk.each do |row|
          imported_index += 1
          test_mode = options["test_mode"]
          return if imported_index > 20 && test_mode
          row[:imported_at] = imported_at
          row[:first_name_last_name_order] = options["first_name_last_name_order"]
{'fraction_non_alphanumeric': 0.08617852735499794, 'fraction_numerical': 0.0034965034965034965, 'mean_word_length': 2.812230497843983, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 60}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6067610', 'n_tokens_mistral': 3030, 'n_tokens_neox': 2953, 'n_words': 583}