<commit_before>module TSPScraper
  class Converter
    # TSP CSV has a janky format:
    #   - Each field in a row, except for the first field, is prepended by a space
    #   - Each row has a final column containing just a space
    # For this reason, we manually clean up the CSV instead of using the Ruby CSV library, especially since we know TSP
    # does not use quotation marks in their CSV fields... at least for now.
    def self.raw_csv_to_hash(raw_csv)
      raw_data = raw_csv.lines.map { |row| row.strip.chomp(',').split(',').map { |field| field.strip } }
      headers = raw_data.shift
      data = []
      raw_data.each do |d|
        hash = {
          date: Date.parse(d.first),
          funds: {}
        }
        headers.each_with_index do |header, index|
          hash[:funds][header] = BigDecimal.new(d[index]) unless index == 0
        end
        data.push(hash)
      end
      data
    end
  end
end<commit_msg>move away from deprecated new() syntax<commit_after>module TSPScraper
  class Converter
    # TSP CSV has a janky format:
    #   - Each field in a row, except for the first field, is prepended by a space
    #   - Each row has a final column containing just a space
    # For this reason, we manually clean up the CSV instead of using the Ruby CSV library, especially since we know TSP
    # does not use quotation marks in their CSV fields... at least for now.
    def self.raw_csv_to_hash(raw_csv)
      raw_data = raw_csv.lines.map { |row| row.strip.chomp(',').split(',').map { |field| field.strip } }
      headers = raw_data.shift
      data = []
      raw_data.each do |d|
        hash = {
          date: Date.parse(d.first),
          funds: {}
        }
        headers.each_with_index do |header, index|
          next if index == 0
          if d[index].empty?
            value = nil
          else
            value = BigDecimal(d[index])
          end
          hash[:funds][header] = value
        end
        data.push(hash)
      end
      data
    end
  end
end
{'fraction_non_alphanumeric': 0.09059405940594059, 'fraction_numerical': 0.0009900990099009901, 'mean_word_length': 2.0949464012251147, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2947139', 'n_tokens_mistral': 618, 'n_tokens_neox': 602, 'n_words': 219}