<commit_before>module Prmd
  def self.combine(paths, options={})
    files = []
    [*paths].each do |path|
      files << if File.directory?(path)
        Dir.glob(File.join(path, '**', '*.json')) +
          Dir.glob(File.join(path, '**', '*.yaml')) -
          [options[:meta]]
      else
        path
      end
    end
    files.flatten!

    # sort for stable loading on any platform
    schemata = []
    files.sort.each do |file|
      begin
        schemata << [file, YAML.load(File.read(file))]
      rescue
        $stderr.puts "unable to parse #{file}"
      end
    end
    unless schemata.length == files.length
      exit(1) # one or more files failed to parse
    end

    data = {<commit_msg>Accept "*.yml" as input in addition to "*.yaml"<commit_after>module Prmd
  def self.combine(paths, options={})
    files = []
    [*paths].each do |path|
      files << if File.directory?(path)
        Dir.glob(File.join(path, '**', '*.json')) +
          Dir.glob(File.join(path, '**', '*.{yaml,yml}')) -
          [options[:meta]]
      else
        path
      end
    end
    files.flatten!

    # sort for stable loading on any platform
    schemata = []
    files.sort.each do |file|
      begin
        schemata << [file, YAML.load(File.read(file))]
      rescue
        $stderr.puts "unable to parse #{file}"
      end
    end
    unless schemata.length == files.length
      exit(1) # one or more files failed to parse
    end

    data = {
{'fraction_non_alphanumeric': 0.1462912087912088, 'fraction_numerical': 0.0013736263736263737, 'mean_word_length': 2.0738396624472575, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3599733', 'n_tokens_mistral': 482, 'n_tokens_neox': 469, 'n_words': 123}