<commit_before>module X2Importer
  # Any class using X2Importer should implement two methods:
  # export_file_name => string pointing to the name of the remote file to parse
  # parse_row => function that describes how to handle each row; takes row as only argument

  def sftp_info_present?
    ENV['SIS_SFTP_HOST'].present? &&
    ENV['SIS_SFTP_USER'].present? &&
    ENV['SIS_SFTP_KEY'].present?
  end

  def import_from_x2
    if sftp_info_present?
      Net::SFTP.start(
        ENV['SIS_SFTP_HOST'],
        ENV['SIS_SFTP_USER'],
        key_data: ENV['SIS_SFTP_KEY'],
        keys_only: true,
        ) do |sftp|
        # For documentation on Net::SFTP::Operations::File, see
        # http://net-ssh.github.io/sftp/v2/api/classes/Net/SFTP/Operations/File.html
        sftp.file.open(export_file_name, "r") do |f|
          parse_for_import(f)
        end
      end
    else
      raise "SFTP information missing"
    end
  end

  def parse_for_import(file)
    require 'csv'
    headers = file.gets.parse_csv.map(&:to_sym)    # Assume headers are first row
    loop do
      next_line = file.gets
      if next_line.present?
        row = Hash[headers.zip(next_line.parse_csv)]
        parse_row row
      else break end
    end
  end
end
<commit_msg>Handle malformed CSV case (multiline descriptions)<commit_after>module X2Importer
  # Any class using X2Importer should implement two methods:
  # export_file_name => string pointing to the name of the remote file to parse
  # parse_row => function that describes how to handle each row; takes row as only argument

  def sftp_info_present?
    ENV['SIS_SFTP_HOST'].present? &&
    ENV['SIS_SFTP_USER'].present? &&
    ENV['SIS_SFTP_KEY'].present?
  end

  def import_from_x2
    if sftp_info_present?
      Net::SFTP.start(
        ENV['SIS_SFTP_HOST'],
        ENV['SIS_SFTP_USER'],
        key_data: ENV['SIS_SFTP_KEY'],
        keys_only: true,
        ) do |sftp|
        # For documentation on Net::SFTP::Operations::File, see
        # http://net-ssh.github.io/sftp/v2/api/classes/Net/SFTP/Operations/File.html
        sftp.file.open(export_file_name, "r") do |f|
          parse_for_import(f)
        end
      end
    else
      raise "SFTP information missing"
    end
  end

  def parse_for_import(file)
    require 'csv'
    headers = file.gets.parse_csv.map(&:to_sym)    # Assume headers are first row
    loop do
      next_line = file.gets
      if next_line.present?
        begin
          next_line.gsub!("\\", '"')             # Force multiline strings to end before linebreak
          csv_row = next_line.parse_csv
          row_with_headers = Hash[headers.zip(csv_row)]
          parse_row row_with_headers
        rescue CSV::MalformedCSVError
        end
      else break end
    end
  end
end

{'fraction_non_alphanumeric': 0.09431245500359971, 'fraction_numerical': 0.0028797696184305254, 'mean_word_length': 2.6184895833333335, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4199566', 'n_tokens_mistral': 938, 'n_tokens_neox': 929, 'n_words': 223}