<commit_before>module Ontology::Distributed
  extend ActiveSupport::Concern

  included do
    acts_as_tree

    def self.homogeneous
      select_with_character_selector('=')
    end

    def self.heterogeneous
      select_with_character_selector('>')
    end

    def self.heterogeneous_children
      ontologies = where("ontologies.type = 'DistributedOntology'").
        select { |ontology| ontology.heterogeneous?}
      where(parent_id: ontologies)
    end

    private
    def self.select_with_character_selector(selector="=")
      stmt = <<-STMT
(
SELECT distributed.id FROM
  (SELECT distributed_count.id, COUNT(distributed_count.id) AS occurrences FROM
    (SELECT (ontologies.parent_id) as id FROM ontologies
    GROUP BY ontologies.logic_id, ontologies.parent_id)
  AS distributed_count
GROUP BY distributed_count.id) AS distributed
WHERE distributed.occurrences #{selector} 1
)
      STMT
      where(type: DistributedOntology).where("ontologies.id IN #{stmt}")
    end

  end

  def distributed?
    is_a? DistributedOntology
  end
<commit_msg>determine heterogeneous children with sql.<commit_after>module Ontology::Distributed
  extend ActiveSupport::Concern

  included do
    acts_as_tree

    def self.homogeneous
      select_with_character_selector('=')
    end

    def self.heterogeneous
      select_with_character_selector('>')
    end

    def self.heterogeneous_children
      where(parent_id: self.heterogeneous)
    end

    private
    def self.select_with_character_selector(selector="=")
      stmt = <<-STMT
(
SELECT distributed.id FROM
  (SELECT distributed_count.id, COUNT(distributed_count.id) AS occurrences FROM
    (SELECT (ontologies.parent_id) as id FROM ontologies
    GROUP BY ontologies.logic_id, ontologies.parent_id)
  AS distributed_count
GROUP BY distributed_count.id) AS distributed
WHERE distributed.occurrences #{selector} 1
)
      STMT
      where(type: DistributedOntology).where("ontologies.id IN #{stmt}")
    end

  end

  def distributed?
    is_a? DistributedOntology
  end

{'fraction_non_alphanumeric': 0.07862407862407862, 'fraction_numerical': 0.0009828009828009828, 'mean_word_length': 3.836104513064133, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2117267', 'n_tokens_mistral': 636, 'n_tokens_neox': 585, 'n_words': 144}