<commit_before>module ManualFm
  def self.readcddb(filename)
    cd = File.read(filename)
    lines = cd.each_line.map{|line|line.chomp}
    until /offsets:$/ === lines.shift
    end
    last_offset = nil
    lens = []
    loop do
      x, offsets = lines.shift.split("\t")
      break if offsets.nil?
      offset = offsets.to_i
      unless last_offset.nil?
        lens << ((offset - last_offset) / 75.0).round
      end
      last_offset = offset
    end
    totallen = lines.shift.split[3].to_i
    lens << totallen - (last_offset / 75.0).round
    until /^DISCID=/ === lines.shift
    end
    data = {}
    loop do
      line = lines.shift
      break if line.nil?<commit_msg>CDDB: handle space and tab<commit_after>module ManualFm
  def self.readcddb(filename)
    cd = File.read(filename)
    lines = cd.each_line.map{|line|line.chomp}
    until /offsets:$/ === lines.shift
    end
    last_offset = nil
    lens = []
    loop do
      x, offsets = lines.shift.split("\s")
      break if offsets.nil?
      offset = offsets.to_i
      unless last_offset.nil?
        lens << ((offset - last_offset) / 75.0).round
      end
      last_offset = offset
    end
    totallen = lines.shift.split[3].to_i
    lens << totallen - (last_offset / 75.0).round
    until /^DISCID=/ === lines.shift
    end
    data = {}
    loop do
      line = lines.shift
      break if line.nil?
{'fraction_non_alphanumeric': 0.12418300653594772, 'fraction_numerical': 0.010167029774872912, 'mean_word_length': 2.1533180778032035, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5284228', 'n_tokens_mistral': 483, 'n_tokens_neox': 464, 'n_words': 109}