<commit_before>require 'git'
require 'nokogiri'
require './page'

class IndexJob
  @queue = :default

  def self.perform(tmpdir, repo)
    clone_repo(repo, tmpdir)
    Dir.chdir "#{tmpdir}/#{repo}" do
      @git_dir.checkout "gh-pages"
      Dir.glob("**/*.html").map(&File.method(:realpath)).each do |html_file|
        relative_path = html_file.match(/#{repo}\/(.+)/)[1]
        html_file_contents = File.read(html_file)

        # TODO: make these configurable
        doc = Nokogiri::HTML(html_file_contents)
        text = doc.xpath("//div[contains(concat(' ',normalize-space(@class),' '),' article-body ')]").text().strip
        title = doc.xpath("//title").text().strip
        last_updated = doc.xpath("//span[contains(concat(' ',normalize-space(@class),' '),'last-modified-at-date')]").strip

        page = Page.new id: "#{repo}::#{relative_path}", title: title, body: text, path: relative_path, last_updated: last_updated

        GitHubPagesSearch::repository.save(page)
      end
    end
  end

  def self.token
    ENV["MACHINE_USER_TOKEN"]
  end

  def self.clone_repo(repo, tmpdir)
    puts "Cloning #{repo}..."
    @git_dir = Git.clone(clone_url_with_token(token, repo), "#{tmpdir}/#{repo}")<commit_msg>See what fetching the whole body looks like<commit_after>require 'git'
require 'nokogiri'
require './page'

class IndexJob
  @queue = :default

  def self.perform(tmpdir, repo)
    clone_repo(repo, tmpdir)
    Dir.chdir "#{tmpdir}/#{repo}" do
      @git_dir.checkout "gh-pages"
      Dir.glob("**/*.html").map(&File.method(:realpath)).each do |html_file|
        relative_path = html_file.match(/#{repo}\/(.+)/)[1]
        html_file_contents = File.read(html_file)

        # TODO: make these configurable?
        doc = Nokogiri::HTML(html_file_contents)
        body = doc.xpath("//body").text().strip
        title = doc.xpath("//title").text().strip
        last_updated = doc.xpath("//span[contains(concat(' ',normalize-space(@class),' '),'last-modified-at-date')]").text().strip

        page = Page.new id: "#{repo}::#{relative_path}", title: title, body: body, path: relative_path, last_updated: last_updated

        GitHubPagesSearch::repository.save(page)
      end
    end
  end

  def self.token
    ENV["MACHINE_USER_TOKEN"]
  end

  def self.clone_repo(repo, tmpdir)
    puts "Cloning #{repo}..."
    @git_dir = Git.clone(clone_url_with_token(token, repo), "#{tmpdir}/#{repo}")
{'fraction_non_alphanumeric': 0.17654372150849565, 'fraction_numerical': 0.0008288437629506838, 'mean_word_length': 3.5121495327102803, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2071025', 'n_tokens_mistral': 826, 'n_tokens_neox': 823, 'n_words': 151}