<commit_before>class ScrapersController < ApplicationController
  before_filter :authenticate_user!, except: [:show, :data]

  def settings
    @scraper = Scraper.find(params[:id])
    unless @scraper.can_write?(current_user)
      redirect_to @scraper
      return
    end
  end

  def new
    @scraper = Scraper.new
  end

  def create
    @scraper = Scraper.new(original_language: params[:scraper][:original_language],
      owner_id: params[:scraper][:owner_id], name: params[:scraper][:name])
    @scraper.full_name = "#{@scraper.owner.to_param}/#{@scraper.name}"
    if !Scraper.can_write?(current_user, @scraper.owner)
      @scraper.errors.add(:owner_id, "doesn't belong to you")
      render :new
    elsif Scraper.exists?(full_name: @scraper.full_name)
      @scraper.errors.add(:name, "is already taken on Morph")
      render :new
    elsif Morph::Github.in_public_use?(@scraper.full_name)
      @scraper.errors.add(:name, "is already taken on GitHub")
      render :new
    else
      repo = Morph::Github.create_repository(current_user, @scraper.owner, @scraper.name)
      # TODO Populate it with a default scraper in the chosen language
      files = {
        Morph::Language.language_to_scraper_filename(@scraper.original_language.to_sym) => Morph::Language.default_scraper(@scraper.original_language.to_sym),
        ".gitignore" => "# Ignore output of scraper\n#{Morph::Database.sqlite_db_filename}\n",
      }
      # TODO Don't use hardcoded urls
      files["README.md"] = "This is a scraper that runs on [Morph](https://morph.io). To get started [see the documentation](https://morph.io/documentation)"
      @scraper.add_commit_to_root_on_github(current_user, files, "Add template for Morph scraper")

      @scraper = Scraper.new_from_github(repo.full_name)
      if @scraper.save        
        # TODO This could be a long running task shouldn't really be in the request cycle
        @scraper.synchronise_repo
        redirect_to @scraper        
      else
        render :new
      end
    end
  end

  def github
    # Get the list of repositories
    @repos = current_user.github_all_public_repos
    @scraper = Scraper.new
  end
<commit_msg>Put a link from the repo back to the Morph scraper<commit_after>class ScrapersController < ApplicationController
  before_filter :authenticate_user!, except: [:show, :data]

  def settings
    @scraper = Scraper.find(params[:id])
    unless @scraper.can_write?(current_user)
      redirect_to @scraper
      return
    end
  end

  def new
    @scraper = Scraper.new
  end

  def create
    @scraper = Scraper.new(original_language: params[:scraper][:original_language],
      owner_id: params[:scraper][:owner_id], name: params[:scraper][:name])
    @scraper.full_name = "#{@scraper.owner.to_param}/#{@scraper.name}"
    if !Scraper.can_write?(current_user, @scraper.owner)
      @scraper.errors.add(:owner_id, "doesn't belong to you")
      render :new
    elsif Scraper.exists?(full_name: @scraper.full_name)
      @scraper.errors.add(:name, "is already taken on Morph")
      render :new
    elsif Morph::Github.in_public_use?(@scraper.full_name)
      @scraper.errors.add(:name, "is already taken on GitHub")
      render :new
    else
      repo = Morph::Github.create_repository(current_user, @scraper.owner, @scraper.name)
      # TODO Populate it with a default scraper in the chosen language
      files = {
        Morph::Language.language_to_scraper_filename(@scraper.original_language.to_sym) => Morph::Language.default_scraper(@scraper.original_language.to_sym),
        ".gitignore" => "# Ignore output of scraper\n#{Morph::Database.sqlite_db_filename}\n",
      }
      # TODO Don't use hardcoded urls
      files["README.md"] = "This is a scraper that runs on [Morph](https://morph.io). To get started [see the documentation](https://morph.io/documentation)"
      @scraper.add_commit_to_root_on_github(current_user, files, "Add template for Morph scraper")

      @scraper = Scraper.new_from_github(repo.full_name)
      if @scraper.save        
        # TODO This could be a long running task shouldn't really be in the request cycle
        repo = current_user.octokit_client.edit_repository(@scraper.full_name, homepage: scraper_url(@scraper))
        @scraper.synchronise_repo
        redirect_to @scraper        
      else
        render :new
      end
    end
  end

  def github
    # Get the list of repositories
    @repos = current_user.github_all_public_repos
    @scraper = Scraper.new
  end

{'fraction_non_alphanumeric': 0.10846854856634808, 'fraction_numerical': 0.0, 'mean_word_length': 3.4687189672293943, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6329584', 'n_tokens_mistral': 1531, 'n_tokens_neox': 1490, 'n_words': 328}