<commit_before>require 'rugged'
require 'yaml'
require 'hardwired'
module DocCollector
  class Branch
    def initialize(repo, branch_name, meta)
      @branch_meta = meta
      @git = repo
      @name = branch_name
      @ref =  @git.branches[name]
      @tree =  @git.lookup(@ref.target_id).tree unless @ref.nil?
      @errors = []
    end 

    attr_accessor :order

    attr_reader :name, :branch_meta, :tree, :errors

    def exists?
      !tree.nil?
    end

    def subfolder
      branch_meta["subfolder"]
    end

    def search_patterns
      @collect_files_config["patterns"] || []
    end 

    def downcase_keys(hash)
      Hash[hash.to_a.map{|p| [p[0].downcase, p[1]]}]
    end 
    def collect_files_array
      (@collect_files_config["files"] || []).map{|h| downcase_keys(h) }

    end 
    def metadata_to_apply
      collect_files_array.select{|set| !set.has_key?("from") }
    end
    def splitting_to_perform
      collect_files_array.select{|set| set.has_key?("render_and_split") }
    end 

    def load_configuration
      return if @collect_files_config
      blob = tree.path("docs/collect_files.yml")
      raise "Branch #{name} is missing docs/collect_files.yml" if blob.nil?
      
      

      text_contents = @git.lookup(blob[:oid]).content
      @collect_files_config = YAML.load(text_contents)
    end

   
    def load_input_files
      return if @input
      file_set = {}
      tree.walk_blobs(:postorder) do |root, e|
        path = root + e[:name]
        
        if search_patterns.any?{ |p| File.fnmatch(p, "/" + path,File::FNM_CASEFOLD) }
          file = {from: path, rawdata: @git.lookup(e[:oid]).read_raw.data, meta: {}}
          file_set[path.downcase] = file
        end
      end

    
      collect_files_array.each do |f|
        f = downcase_keys(f)
        from = f["from"]
        blob = tree.path(from)
        if blob.nil?
          errors << "Branch #{name} does not contain #{from}"
          next
        end<commit_msg>If a file is missing, don't crash (as much)<commit_after>require 'rugged'
require 'yaml'
require 'hardwired'
module DocCollector
  class Branch
    def initialize(repo, branch_name, meta)
      @branch_meta = meta
      @git = repo
      @name = branch_name
      @ref =  @git.branches[name]
      @tree =  @git.lookup(@ref.target_id).tree unless @ref.nil?
      @errors = []
    end 

    attr_accessor :order

    attr_reader :name, :branch_meta, :tree, :errors

    def exists?
      !tree.nil?
    end

    def subfolder
      branch_meta["subfolder"]
    end

    def search_patterns
      @collect_files_config["patterns"] || []
    end 

    def downcase_keys(hash)
      Hash[hash.to_a.map{|p| [p[0].downcase, p[1]]}]
    end 
    def collect_files_array
      (@collect_files_config["files"] || []).map{|h| downcase_keys(h) }

    end 
    def metadata_to_apply
      collect_files_array.select{|set| !set.has_key?("from") }
    end
    def splitting_to_perform
      collect_files_array.select{|set| set.has_key?("render_and_split") }
    end 

    def load_configuration
      return if @collect_files_config
      yaml_path = "docs/collect_files.yml"
      blob = tree.path(yaml_path)
      raise "Branch #{name} is missing docs/collect_files.yml" if blob.nil?
      
      

      text_contents = @git.lookup(blob[:oid]).content
      begin 
        @collect_files_config = YAML.load(text_contents)
      rescue Psych::SyntaxError
        @errors << "Invalid YAML in #{yaml_path}, branch #{name} \n #{$!}"
        puts @errors
      end
    end

   
    def load_input_files
      return if @input
      file_set = {}
      tree.walk_blobs(:postorder) do |root, e|
        path = root + e[:name]
        
        if search_patterns.any?{ |p| File.fnmatch(p, "/" + path,File::FNM_CASEFOLD) }
          file = {from: path, rawdata: @git.lookup(e[:oid]).read_raw.data, meta: {}}
          file_set[path.downcase] = file
        end
      end

    
      collect_files_array.each do |f|
        f = downcase_keys(f)
        from = f["from"]
        begin 
          blob = tree.path(from)
        rescue Rugged::TreeError
          errors << "Branch #{name} does not contain #{from}\n #{$!}"
          next
        end

{'fraction_non_alphanumeric': 0.12170192536249108, 'fraction_numerical': 0.0009507962918944616, 'mean_word_length': 2.388083735909823, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2459469', 'n_tokens_mistral': 1436, 'n_tokens_neox': 1386, 'n_words': 298}