<commit_before>require "net/http"
require "uri"
require "json"

module Setup
  class Concepts
    class_attribute :concepts

    def initialize
      self.concepts = []
    end

    def fetch_concepts
      uri = URI.parse("https://staging.quill.org/api/v1/concepts")
      response = Net::HTTP.get_response(uri)
      self.concepts = self.string_to_json(response.body)["concepts"]
      return response
    end

    def find_parent_concept_from_concepts(child)
      concepts.select{|con| con["id"] == child["parent_id"]}.first
    end

    def find_or_create_parent_in_db(child)
      found_parent = self.find_parent_concept_from_concepts(child)
      parent = Concept.find_by_name(found_parent["name"])
      if parent.nil?
        parent = create_concept(found_parent)
      end
      return parent
    end

    def create_concept(concept)
      con = Concept.new(name: concept["name"], uid: concept["uid"])
      if concept["parent_id"]
        parent = self.find_or_create_parent_in_db(concept) 
        con.parent_id = parent.id
      end
      con.save
      return con
    end

    def create_all
      self.concepts.each do |con|
        self.create_concept(con)
      end
    end

    def string_to_json(response_string)
      JSON.parse(response_string)
    end
  end
end <commit_msg>add a run function to fetch and then create the concepts.<commit_after>require "net/http"
require "uri"
require "json"

module Setup
  class Concepts
    class_attribute :concepts

    def initialize
      self.concepts = []
    end

    def fetch_concepts
      uri = URI.parse("https://staging.quill.org/api/v1/concepts")
      response = Net::HTTP.get_response(uri)
      self.concepts = self.string_to_json(response.body)["concepts"]
      return response
    end

    def find_parent_concept_from_concepts(child)
      concepts.select{|con| con["id"] == child["parent_id"]}.first
    end

    def find_or_create_parent_in_db(child)
      found_parent = self.find_parent_concept_from_concepts(child)
      parent = Concept.find_by_name(found_parent["name"])
      if parent.nil?
        parent = create_concept(found_parent)
      end
      return parent
    end

    def create_concept(concept)
      con = Concept.new(name: concept["name"], uid: concept["uid"])
      if concept["parent_id"]
        parent = self.find_or_create_parent_in_db(concept) 
        con.parent_id = parent.id
      end
      con.save
      return con
    end

    def create_all
      self.concepts.each do |con|
        self.create_concept(con)
      end
    end

    def string_to_json(response_string)
      JSON.parse(response_string)
    end

    def run
      self.fetch_concepts
      self.create_all
    end
  end
end 
{'fraction_non_alphanumeric': 0.09205175600739371, 'fraction_numerical': 0.0007393715341959334, 'mean_word_length': 2.7793296089385473, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1266863', 'n_tokens_mistral': 906, 'n_tokens_neox': 849, 'n_words': 173}