<commit_before>class Tracksperanto::Import::Boujou < Tracksperanto::Import::Base
  
  def self.human_name
    "Boujou feature tracks export"
  end
  
  def parse(io)
    wrapped_io = Tracksperanto::ExtIO.new(io)
    detect_columns(wrapped_io)
    trackers = {}
    extract_trackers(wrapped_io) do | k |
      name, frame, x, y = k["track_id"], k["view"], k["x"], k["y"]
      trackers[name] ||= Tracksperanto::Tracker.new(:name => name)
      report_progress("Extracting frame #{frame} of #{name}")
      trackers[name].keyframe!(:frame => (frame.to_i - 1), :abs_y => (@height.to_f - y.to_f), :abs_x => x)
    end
    trackers.values.sort{|a,b| a.name <=> b.name }
  end
  
  private
  
  COMMENT = /^# /
  
  def detect_columns(io)
    until io.eof? do 
      line = io.gets_and_strip
      if line =~ /^# track_id/
        report_progress("Detecting columns")
        return set_columns_from(line)
      end
    end
  end
  
  def set_columns_from(line)
    @columns = line.gsub(COMMENT, '').split
  end
  
  #
  #
  # # track_id    view      x    y
  # Target_track_1  5  252.046  171.677
  def extract_trackers(io)
    until io.eof?
      line = io.gets_and_strip
      next if comment?(line)
      yield(make_column_hash(line))
    end
  end
  
  def make_column_hash(line)
    Hash[*@columns.zip(line.split).flatten]
  end
  
  def comment?(line)
    line =~ COMMENT
  end
end<commit_msg>Yield unpacked values to the block in the Boujou importer<commit_after>class Tracksperanto::Import::Boujou < Tracksperanto::Import::Base
  
  def self.human_name
    "Boujou feature tracks export"
  end
  
  def parse(io)
    wrapped_io = Tracksperanto::ExtIO.new(io)
    detect_columns(wrapped_io)
    trackers = {}
    filtering_trackers_from(wrapped_io) do | name, frame, x, y |
      trackers[name] ||= Tracksperanto::Tracker.new(:name => name)
      report_progress("Extracting frame #{frame} of #{name}")
      trackers[name].keyframe!(:frame => (frame.to_i - 1), :abs_y => (@height.to_f - y.to_f), :abs_x => x)
    end
    trackers.values.sort{|a,b| a.name <=> b.name }
  end
  
  private
  
  COMMENT = /^# /
  
  def detect_columns(io)
    until io.eof? do 
      line = io.gets_and_strip
      if line =~ /^# track_id/
        report_progress("Detecting columns")
        return set_columns_from(line)
      end
    end
  end
  
  def set_columns_from(line)
    @columns = line.gsub(COMMENT, '').split
  end
  
  #
  #
  # # track_id    view      x    y
  # Target_track_1  5  252.046  171.677
  def filtering_trackers_from(io)
    until io.eof?
      line = io.gets_and_strip
      next if comment?(line)
      column = make_column_hash(line)
      yield(column["track_id"], column["view"], column["x"], column["y"])
    end
  end
  
  def make_column_hash(line)
    Hash[*@columns.zip(line.split).flatten]
  end
  
  def comment?(line)
    line =~ COMMENT
  end
end
{'fraction_non_alphanumeric': 0.13026819923371646, 'fraction_numerical': 0.01044932079414838, 'mean_word_length': 2.7889182058047495, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3472951', 'n_tokens_mistral': 1093, 'n_tokens_neox': 1073, 'n_words': 214}