<commit_before>require 'kolekt/sources/base'

module Kolekt::Sources
  class Augeas < Base
    def self.active?
      Kolekt::Helpers::Require.can_require? %w[augeas]
    end

    def collect
      require 'augeas'

      ::Augeas::open '/', nil, ::Augeas::NO_LOAD do |aug|
        # Those lenses create too much noise or are somewhat broken for us
        %w[Services Protocols Xml].each do |lens|
          aug.rm "/augeas/load/#{lens}"
        end

        aug.load
        return [true, makeHash(aug, '/files')]
      end
    end

    private
    def mix h, v, cpath
      v_relevant = !v.nil? and v != cpath
      h_relevant = !h.nil? and !h.empty?

      case true
      when (v_relevant and h_relevant) then
        raise 'Damn, $IT happened!' if h[:key] != nil
        h[:key] = v
        return h
      when h_relevant then
        return h
      when v_relevant then
        return v
      end
    end

    private
    def makeHash aug, path
      escaped = path.gsub '!', '\!'
      children = aug.match "#{escaped}/*[label()!=\"#comment\"]"

      return nil if children.empty?

      if children.all? {|c| c =~ /\/\d+$/} # array
        res = []
        children.each do |cpath|
          res[File.basename(cpath).to_i - 1] = makeHash aug, cpath
        end
        return res
      end

      res = {}
      children.each do |cpath|
        name = File.basename cpath
        v = aug.get cpath
        h = makeHash aug, cpath

        if name =~ /(.*)\[(\d+)\]/
          res[$1] ||= []
          res[$1][$2.to_i - 1] = mix(h, v, cpath)
        else
          res[name] = mix h, v, cpath
        end
      end

      return res
    end
  end
end
<commit_msg>Sources::Augeas: much simpler code, doesn't seem to miss anything now<commit_after>require 'kolekt/sources/base'

module Kolekt::Sources
  class Augeas < Base
    def self.active?
      Kolekt::Helpers::Require.can_require? %w[augeas]
    end

    def collect
      require 'augeas'

      ::Augeas::open '/', nil, ::Augeas::NO_LOAD do |aug|
        # Those lenses create too much noise or are somewhat broken for us
        %w[Services Protocols Xml].each do |lens|
          aug.rm "/augeas/load/#{lens}"
        end

        aug.load
        return [true, makeHash(aug, '/files')]
      end
    end

    private
    def makeHash aug, path
      escaped = path.gsub '!', '\!'
      children = aug.match "#{escaped}/*[label()!=\"#comment\"]"
      value = aug.get escaped

      return value if children.empty?

      if children.all? {|c| c =~ /\/\d+$/} and value.nil? # array
        res = []
        children.each do |cpath|
          id = File.basename(cpath).to_i - 1
          raise '0 index!' if id == -1
          res[id] = makeHash aug, cpath
        end
        return res
      end

      res = {}

      children.each do |cpath|
        name = File.basename cpath

        if name =~ /(.*)\[(\d+)\]/
          res[$1] ||= []
          res[$1][$2.to_i - 1] = makeHash aug, cpath
        else
          res[name] = makeHash aug, cpath
        end
      end

      if !value.nil? and value != path
        raise 'Damn it, a conflict' if res.has_key? 'key'
        res['key'] = value
      end

      return res
    end
  end
end

{'fraction_non_alphanumeric': 0.13080037371535347, 'fraction_numerical': 0.003737153534724385, 'mean_word_length': 1.7476475620188194, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1526612', 'n_tokens_mistral': 1096, 'n_tokens_neox': 1048, 'n_words': 309}