<commit_before>def model_attributes(table)
  attributes = table.rows_hash.dup

  attributes.dup.each do |key, value|
    case value
    when /\A\d+\Z/
      # Convert integer
      attributes[key] = value.to_i
    end

    # Transform
    #  | Schedule[aimed]#Arrival   | 2017-01-01T13:00:00.000Z          |
    #  | Schedule[aimed]#Departure | 2017-01-01T13:02:00.000Z          |
    # into
    # "Schedules" => [
    #   {"Kind"=>"aimed", "ArrivalTime"=>"2017-01-01T13:00:00.000Z", "DepartureTime"=>"2017-01-01T13:02:00.000Z"}
    # ]
    if key =~ /Schedule\[(aimed|expected|actual)\]#(Arrival|Departure)/
      schedule_type = $1
      attribute = $2

      attributes["Schedules"] ||= []

      schedule = attributes["Schedules"].find { |s| s["Kind"] == schedule_type }
      unless schedule
        schedule = { "Kind" => schedule_type }
        attributes["Schedules"] << schedule
      end

      schedule["#{attribute}Time"] = value

      attributes.delete key
    end

    if key =~ /Attribute\[([^\]]+)\]/
      name = $1
      attributes["Attributes"] ||= {}
      attributes["Attributes"][name] = value
      attributes.delete key
    end    
  end

  if objectids = (attributes["ObjectIDs"] || attributes["ObjectIDs"])
    attributes["ObjectIDs"] = JSON.parse("{ #{objectids} }")
  end

  attributes
end

def api_attributes(json)
  attributes = JSON.parse(json)

  puts attributes.inspect

  objectids = attributes["ObjectIDs"]
  if Array === objectids
    attributes["ObjectIDs"] = Hash[objectids.map { |objectid| [objectid["Kind"], objectid["Value"]] }]
  end

  attributes
end<commit_msg>Remove a nice puts<commit_after>def model_attributes(table)
  attributes = table.rows_hash.dup

  attributes.dup.each do |key, value|
    case value
    when /\A\d+\Z/
      # Convert integer
      attributes[key] = value.to_i
    end

    # Transform
    #  | Schedule[aimed]#Arrival   | 2017-01-01T13:00:00.000Z          |
    #  | Schedule[aimed]#Departure | 2017-01-01T13:02:00.000Z          |
    # into
    # "Schedules" => [
    #   {"Kind"=>"aimed", "ArrivalTime"=>"2017-01-01T13:00:00.000Z", "DepartureTime"=>"2017-01-01T13:02:00.000Z"}
    # ]
    if key =~ /Schedule\[(aimed|expected|actual)\]#(Arrival|Departure)/
      schedule_type = $1
      attribute = $2

      attributes["Schedules"] ||= []

      schedule = attributes["Schedules"].find { |s| s["Kind"] == schedule_type }
      unless schedule
        schedule = { "Kind" => schedule_type }
        attributes["Schedules"] << schedule
      end

      schedule["#{attribute}Time"] = value

      attributes.delete key
    end

    if key =~ /Attribute\[([^\]]+)\]/
      name = $1
      attributes["Attributes"] ||= {}
      attributes["Attributes"][name] = value
      attributes.delete key
    end
  end

  if objectids = (attributes["ObjectIDs"] || attributes["ObjectIDs"])
    attributes["ObjectIDs"] = JSON.parse("{ #{objectids} }")
  end

  attributes
end

def api_attributes(json)
  attributes = JSON.parse(json)

  objectids = attributes["ObjectIDs"]
  if Array === objectids
    attributes["ObjectIDs"] = Hash[objectids.map { |objectid| [objectid["Kind"], objectid["Value"]] }]
  end

  attributes
end
{'fraction_non_alphanumeric': 0.16227244193345888, 'fraction_numerical': 0.044569993722536096, 'mean_word_length': 2.9248768472906406, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3372292', 'n_tokens_mistral': 1151, 'n_tokens_neox': 1036, 'n_words': 188}