<commit_before>module Idhja22
  class Tree
    attr_accessor :root
    class << self
      def train(dataset)
        new(dataset, dataset.attribute_labels)
      end

      def train_from_csv(filename)
        ds = Dataset.from_csv(filename)
        train(ds)
      end

      def build_node(dataset, attributes_available, parent_probability = nil)
        if(dataset.size < 10)
          if(parent_probability)
            return Idhja22::LeafNode.new(parent_probability)
          else
            return nil
          end
        end

        #if successful termination - create and return a leaf node
        if(dataset.terminating?)
          return Idhja22::LeafNode.new(dataset.probability)
        end

        #if we have no more attributes left to split the dataset on, then return a leafnode
        if(attributes_available.empty?)
          return Idhja22::LeafNode.new(dataset.probability)
        end

        data_split , best_attribute = best_attribute(dataset, attributes_available)

        node = Idhja22::DecisionNode.new(data_split, best_attribute, attributes_available-[best_attribute], dataset.probability)
        #could be that although there's attributes left to split on, we don't get anything useful out, in which case ignore the split and terminate the tree
        if node.branches.empty?
          return Idhja22::LeafNode.new(dataset.probability)
        end

        return node
      end

      private
      def best_attribute(dataset, attributes_available)
        data_split = best_attribute = nil
        igain = - Float::INFINITY

        attributes_available.each do |attr|
          possible_split = dataset.split(dataset.attribute_labels.index(attr))
          possible_igain = dataset.entropy
          possible_split.each do |value, ds|
            possible_igain -= (ds.size.to_f/dataset.size.to_f)*ds.entropy
          end
          if(possible_igain > igain)
            igain = possible_igain
            data_split = possible_split
            best_attribute = attr
          end
        end
        return data_split, best_attribute
      end
    end

    def initialize(dataset, attributes_available)
      @root = self.class.build_node(dataset, attributes_available)
    end

    def get_rules
      output = "if "
      rules = root.get_rules
      "if " + rules.join("\nelsif ")
    end
  end
end<commit_msg>removed an unused line<commit_after>module Idhja22
  class Tree
    attr_accessor :root
    class << self
      def train(dataset)
        new(dataset, dataset.attribute_labels)
      end

      def train_from_csv(filename)
        ds = Dataset.from_csv(filename)
        train(ds)
      end

      def build_node(dataset, attributes_available, parent_probability = nil)
        if(dataset.size < 10)
          if(parent_probability)
            return Idhja22::LeafNode.new(parent_probability)
          else
            return nil
          end
        end

        #if successful termination - create and return a leaf node
        if(dataset.terminating?)
          return Idhja22::LeafNode.new(dataset.probability)
        end

        #if we have no more attributes left to split the dataset on, then return a leafnode
        if(attributes_available.empty?)
          return Idhja22::LeafNode.new(dataset.probability)
        end

        data_split , best_attribute = best_attribute(dataset, attributes_available)

        node = Idhja22::DecisionNode.new(data_split, best_attribute, attributes_available-[best_attribute], dataset.probability)
        #could be that although there's attributes left to split on, we don't get anything useful out, in which case ignore the split and terminate the tree
        if node.branches.empty?
          return Idhja22::LeafNode.new(dataset.probability)
        end

        return node
      end

      private
      def best_attribute(dataset, attributes_available)
        data_split = best_attribute = nil
        igain = - Float::INFINITY

        attributes_available.each do |attr|
          possible_split = dataset.split(dataset.attribute_labels.index(attr))
          possible_igain = dataset.entropy
          possible_split.each do |value, ds|
            possible_igain -= (ds.size.to_f/dataset.size.to_f)*ds.entropy
          end
          if(possible_igain > igain)
            igain = possible_igain
            data_split = possible_split
            best_attribute = attr
          end
        end
        return data_split, best_attribute
      end
    end

    def initialize(dataset, attributes_available)
      @root = self.class.build_node(dataset, attributes_available)
    end

    def get_rules
      rules = root.get_rules
      "if " + rules.join("\nelsif ")
    end
  end
end
{'fraction_non_alphanumeric': 0.06724649978786593, 'fraction_numerical': 0.005939753924480272, 'mean_word_length': 2.3018207282913163, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2442152', 'n_tokens_mistral': 1325, 'n_tokens_neox': 1279, 'n_words': 332}