<commit_before>require 'net/https'
require 'json'

class ProjectsWorker
  include Sidekiq::Worker

  REPO    = 'HeisenBugDev/HeisenBugDev-content'
  BRANCH  = 'master'
  FILE    = 'projects.json'
  GITHUB  = URI("https://raw.github.com/")
  URN     = "/#{REPO}/#{BRANCH}/#{FILE}"


  def request_file
    http              = Net::HTTP.new(GITHUB.host, GITHUB.port)
    http.use_ssl      = true
    http.verify_mode  = OpenSSL::SSL::VERIFY_NONE
    request           = Net::HTTP::Get.new(URN)
    response          = http.request(request)
  end

  def file_string
    request_file.body.to_s
  end

  def data_from_file
    JSON.parse(file_string)
  end

  def perform
    data = data_from_file

    data["projects"].each do |key|
      puts Project.exists?(:name => key[:name.to_s])
    end
  end
end<commit_msg>Super simple updater<commit_after>require 'net/https'
require 'json'

class ProjectsWorker
  include Sidekiq::Worker

  REPO      = 'HeisenBugDev/HeisenBugDev-content'
  BRANCH    = 'master'
  FILE      = 'projects.json'
  GITHUB    = URI("https://raw.github.com/")
  URN       = "/#{REPO}/#{BRANCH}/#{FILE}"
  @project  = nil

  def request_file
    http              = Net::HTTP.new(GITHUB.host, GITHUB.port)
    http.use_ssl      = true
    http.verify_mode  = OpenSSL::SSL::VERIFY_NONE
    request           = Net::HTTP::Get.new(URN)
    response          = http.request(request)
  end

  def file_string
    request_file.body.to_s
  end

  def data_from_file
    JSON.parse(file_string)
  end

  def perform
    data = data_from_file

    data["projects"].each do |key|
      name = key['name']
      if Project.exists?(:name => name)
        @project = Project.find_by_name(name)
      else
        @project = Project.new(:name => name)
      end
      @project.update(key)
    end
  end
end
{'fraction_non_alphanumeric': 0.12278761061946902, 'fraction_numerical': 0.0, 'mean_word_length': 2.505813953488372, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5500306', 'n_tokens_mistral': 647, 'n_tokens_neox': 630, 'n_words': 119}