<commit_before># You'll need to require these if you
# want to develop while running with ruby.
# The config/rackup.ru requires these as well
# for it's own reasons.
#
# $ ruby heroku-sinatra-app.rb
#
require 'rubygems'
require 'sinatra'
require 'nokogiri'
require 'net/http'
require 'open-uri'

configure :production do
end

# Quick test
get '/' do
  content_type :text
  uri = params[:url]
  selector = params[:selector]
  callback = params[:callback]
  unless uri.nil?
    url = URI::parse(uri.gsub(' ','+'))
    http = Net::HTTP.new(url.host)
    path = (url.query == '' || url.query.nil?) ? ((url.path == '' || url.path.nil?) ? '/' : url.path) : "#{url.path}?#{url.query}"
    resp, body = http.get2(path, {'User-Agent' => 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.1) Gecko/20060111 Firefox/1.5.0.1'})
    return(body) if selector.nil?
    puts selector
    doc = Nokogiri::HTML(body)
    output = doc.css(selector).to_s
    return(output) if callback.nil?
    content_type :js
    return("#{callback}('#{output.gsub("'","\\'").gsub("\n",'\n')}')")
  end
  content_type :html
  haml :index
end

# Test at <appname>.heroku.com

# You can see all your app specific information this way.
# IMPORTANT! This is a very bad thing to do for a production
# application with sensitive information

get '/env' do
  ENV.inspect
end
<commit_msg>Remove the env inspection endpoint<commit_after># You'll need to require these if you
# want to develop while running with ruby.
# The config/rackup.ru requires these as well
# for it's own reasons.
#
# $ ruby heroku-sinatra-app.rb
#
require 'rubygems'
require 'sinatra'
require 'nokogiri'
require 'net/http'
require 'open-uri'

configure :production do
end

# Quick test
get '/' do
  content_type :text
  uri = params[:url]
  selector = params[:selector]
  callback = params[:callback]
  unless uri.nil?
    url = URI::parse(uri.gsub(' ','+'))
    http = Net::HTTP.new(url.host)
    path = (url.query == '' || url.query.nil?) ? ((url.path == '' || url.path.nil?) ? '/' : url.path) : "#{url.path}?#{url.query}"
    resp, body = http.get2(path, {'User-Agent' => 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.1) Gecko/20060111 Firefox/1.5.0.1'})
    return(body) if selector.nil?
    puts selector
    doc = Nokogiri::HTML(body)
    output = doc.css(selector).to_s
    return(output) if callback.nil?
    content_type :js
    return("#{callback}('#{output.gsub("'","\\'").gsub("\n",'\n')}')")
  end
  content_type :html
  haml :index
end

{'fraction_non_alphanumeric': 0.17353058776489405, 'fraction_numerical': 0.01679328268692523, 'mean_word_length': 3.659217877094972, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7205101', 'n_tokens_mistral': 929, 'n_tokens_neox': 905, 'n_words': 240}