<commit_before>class ScrapersController < ApplicationController
  before_filter :authenticate_user!, except: [:show, :data, :run_remote]

  # The run_remote method will be secured with a key so shouldn't need csrf token authentication
  skip_before_filter :verify_authenticity_token, :only => [:run_remote]
  
  def new
    # Get the list of repositories
    # TODO Move this to an initializer
    Octokit.auto_paginate = true
    @repos = current_user.octokit_client.repositories(nil, sort: :pushed)
    @scraper = Scraper.new
  end

  def create
    # Look up the repository by name
    repo = current_user.octokit_client.repository("#{current_user.to_param}/#{params[:scraper][:name]}")

    # Populate a new scraper with information from the repo
    @scraper = Scraper.new(name: repo.name, full_name: repo.full_name,
      description: repo.description, github_id: repo.id, owner_id: current_user.id,
      github_url: repo.rels[:html].href, git_url: repo.rels[:git].href)
    if @scraper.save
      redirect_to @scraper
    else
      render :new
    end
  end

  def show
    @scraper = Scraper.find(params[:id])
    @rows = @scraper.database.first_ten_rows
  end

  def destroy
    @scraper = Scraper.find(params[:id])
    if @scraper.can_write?(current_user)
      flash[:notice] = "Scraper #{@scraper.name} deleted"
      @scraper.destroy
      # TODO Make this done by default after calling Scraper#destroy
      @scraper.destroy_repo_and_data
      redirect_to current_user
    else
      flash[:alert] = "Can't delete someone else's scraper!"
      redirect_to @scraper
    end
  end

  def update
    @scraper = Scraper.find(params[:id])
    if @scraper.can_write?(current_user)
      # TODO This is definitely the dumb and long winded way to do things
      if @scraper.update_attributes(auto_run: params[:scraper][:auto_run])
        flash[:notice] = "Scraper settings successfully updated"
      end
    else
      flash[:alert] = "Can't update someone else's scraper"
    end
    redirect_to @scraper
  end

  def run
    scraper = Scraper.find(params[:id])
    if scraper.can_write?(current_user)
      scraper.queue!
    else
      flash[:alert] = "Can't run someone else's scraper!"
    end
    redirect_to scraper
  end

  # Receive code from a remote client, run it and return the result.
  # This will be a long running request
  def run_remote
    puts "In run_remote"
    render text: "Well done!\n"
    p params
    code = params[:code]
    File.write("/tmp/foobar.tar", code)
    puts "Files written to uploaded_files"
    Archive::Tar::Minitar.unpack('/tmp/foobar.tar', 'uploaded_files')
  end

  # TODO Extract checking of who owns the scraper
  def clear
    scraper = Scraper.find(params[:id])
    if scraper.can_write?(current_user)
      scraper.database.clear
    else
      flash[:alert] = "Can't clear someone else's scraper!"
    end
    redirect_to scraper    
  end

  def data
    scraper = Scraper.find(params[:id])
    if params[:format] == "sqlite"
      send_file scraper.database.sqlite_db_path, filename: "#{scraper.name}.sqlite",
        type: "application/x-sqlite3"
    else
      query = params[:query] || scraper.database.select_all
      begin
        rows = scraper.database.sql_query(query)
        respond_to do |format|
          format.json { render :json => rows}
          format.csv do
            csv_string = CSV.generate do |csv|
              csv << rows.first.keys unless rows.empty?
              rows.each do |row|
                csv << row.values
              end
            end
            send_data csv_string, :filename => "#{scraper.name}.csv"
          end
        end
      rescue SQLite3::Exception => e
        respond_to do |format|
          format.json { render :json => {error: e.to_s} }
          format.csv { send_data "error: #{e}", :filename => "#{scraper.name}.csv" }
        end
      end
    end
  end
end
<commit_msg>Simplify now that sending in more sane format<commit_after>class ScrapersController < ApplicationController
  before_filter :authenticate_user!, except: [:show, :data, :run_remote]

  # The run_remote method will be secured with a key so shouldn't need csrf token authentication
  skip_before_filter :verify_authenticity_token, :only => [:run_remote]
  
  def new
    # Get the list of repositories
    # TODO Move this to an initializer
    Octokit.auto_paginate = true
    @repos = current_user.octokit_client.repositories(nil, sort: :pushed)
    @scraper = Scraper.new
  end

  def create
    # Look up the repository by name
    repo = current_user.octokit_client.repository("#{current_user.to_param}/#{params[:scraper][:name]}")

    # Populate a new scraper with information from the repo
    @scraper = Scraper.new(name: repo.name, full_name: repo.full_name,
      description: repo.description, github_id: repo.id, owner_id: current_user.id,
      github_url: repo.rels[:html].href, git_url: repo.rels[:git].href)
    if @scraper.save
      redirect_to @scraper
    else
      render :new
    end
  end

  def show
    @scraper = Scraper.find(params[:id])
    @rows = @scraper.database.first_ten_rows
  end

  def destroy
    @scraper = Scraper.find(params[:id])
    if @scraper.can_write?(current_user)
      flash[:notice] = "Scraper #{@scraper.name} deleted"
      @scraper.destroy
      # TODO Make this done by default after calling Scraper#destroy
      @scraper.destroy_repo_and_data
      redirect_to current_user
    else
      flash[:alert] = "Can't delete someone else's scraper!"
      redirect_to @scraper
    end
  end

  def update
    @scraper = Scraper.find(params[:id])
    if @scraper.can_write?(current_user)
      # TODO This is definitely the dumb and long winded way to do things
      if @scraper.update_attributes(auto_run: params[:scraper][:auto_run])
        flash[:notice] = "Scraper settings successfully updated"
      end
    else
      flash[:alert] = "Can't update someone else's scraper"
    end
    redirect_to @scraper
  end

  def run
    scraper = Scraper.find(params[:id])
    if scraper.can_write?(current_user)
      scraper.queue!
    else
      flash[:alert] = "Can't run someone else's scraper!"
    end
    redirect_to scraper
  end

  # Receive code from a remote client, run it and return the result.
  # This will be a long running request
  def run_remote
    render text: "Well done!\n"
    Archive::Tar::Minitar.unpack(params[:code].tempfile, 'uploaded_files')
  end

  # TODO Extract checking of who owns the scraper
  def clear
    scraper = Scraper.find(params[:id])
    if scraper.can_write?(current_user)
      scraper.database.clear
    else
      flash[:alert] = "Can't clear someone else's scraper!"
    end
    redirect_to scraper    
  end

  def data
    scraper = Scraper.find(params[:id])
    if params[:format] == "sqlite"
      send_file scraper.database.sqlite_db_path, filename: "#{scraper.name}.sqlite",
        type: "application/x-sqlite3"
    else
      query = params[:query] || scraper.database.select_all
      begin
        rows = scraper.database.sql_query(query)
        respond_to do |format|
          format.json { render :json => rows}
          format.csv do
            csv_string = CSV.generate do |csv|
              csv << rows.first.keys unless rows.empty?
              rows.each do |row|
                csv << row.values
              end
            end
            send_data csv_string, :filename => "#{scraper.name}.csv"
          end
        end
      rescue SQLite3::Exception => e
        respond_to do |format|
          format.json { render :json => {error: e.to_s} }
          format.csv { send_data "error: #{e}", :filename => "#{scraper.name}.csv" }
        end
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.10040212738357764, 'fraction_numerical': 0.0005188740433259826, 'mean_word_length': 2.777560019598236, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3417168', 'n_tokens_mistral': 2513, 'n_tokens_neox': 2456, 'n_words': 655}