<commit_before>require 'sinatra'
require 'haml'
require 'restclient'
require 'json'

require 'open-uri'

DOWNLOAD_DIR = File.join(File.dirname(__FILE__), 'tmp', 'download')

module GemWatch
  class << self
    attr_accessor :assets_path, :prefix
  end
  self.assets_path = ""
  self.prefix = ""
  def asset_url(url)
    GemWatch.assets_path + url
  end
  def app_url(url)
    GemWatch.prefix + url
  end
end

class GemWatch::Gem
  class NotFound < Exception; end
  class CommandFailed < Exception; end
  def initialize(gemname, wanted_version = nil)
    begin
      @data = JSON.parse(RestClient.get("http://rubygems.org/api/v1/gems/#{gemname}.json").body)
    rescue RestClient::ResourceNotFound
      raise GemWatch::Gem::NotFound
    end
    if wanted_version && wanted_version != version
      raise GemWatch::Gem::NotFound
    end
  end
  def name
    @data['name']
  end
  def version
    @data['version']
  end
  def uri
    @data['gem_uri']
  end
  def info
    @data['info']
  end
  def gem
    File.basename(uri)
  end
  def run(cmd)
    system(cmd)
    if $? && ($? >> 8) > 0
      raise GemWatch::Gem::CommandFailed, "[#{cmd} failed!]"
    end
  end
  def download(uri)
    open(uri, 'rb') do |downloaded_gem|
      File.open(gem, 'wb') do |file|
        file.write(downloaded_gem.read)
      end
    end
  end
  def download_and_convert!
    unless File.exist?(tarball_path)
      FileUtils.rm_rf(absolute_directory)
      FileUtils.mkdir_p(absolute_directory)
      Dir.chdir(absolute_directory) do
        download uri
        run "tar xfm #{gem}"
        run "tar xzfm data.tar.gz"
        run "zcat metadata.gz > metadata.yml"
        FileUtils.rm_f([gem, "data.tar.gz", "metadata.gz"])
      end
      Dir.chdir(File.dirname(absolute_directory)) do
        run "tar czf #{tarball} #{directory}"
        FileUtils.rm_rf(directory)
      end
    end
  end
  def directory
    "#{name}-#{version}"
  end
  def absolute_directory
    File.join(DOWNLOAD_DIR, directory)
  end
  def tarball
    "#{directory}.tar.gz"
  end
  def tarball_uri
    "/#{File.basename(DOWNLOAD_DIR)}/#{tarball}"
  end
  def tarball_path
    File.join(DOWNLOAD_DIR, tarball)
  end
end
module HostHelper
  def host_with_port
    if request.respond_to?(:host_with_port)
      request.host_with_port
    else
      request.host + ([80,443].include?(request.port) ? '' : (':' + request.port.to_s))
    end
  end<commit_msg>Better organization of cached tarballs<commit_after>require 'sinatra'
require 'haml'
require 'restclient'
require 'json'

require 'open-uri'

DOWNLOAD_BASE = File.join(File.dirname(__FILE__), 'tmp', 'download')

module GemWatch
  class << self
    attr_accessor :assets_path, :prefix
  end
  self.assets_path = ""
  self.prefix = ""
  def asset_url(url)
    GemWatch.assets_path + url
  end
  def app_url(url)
    GemWatch.prefix + url
  end
end

class GemWatch::Gem
  class NotFound < Exception; end
  class CommandFailed < Exception; end
  def initialize(gemname, wanted_version = nil)
    begin
      @data = JSON.parse(RestClient.get("http://rubygems.org/api/v1/gems/#{gemname}.json").body)
    rescue RestClient::ResourceNotFound
      raise GemWatch::Gem::NotFound
    end
    if wanted_version && wanted_version != version
      raise GemWatch::Gem::NotFound
    end
  end
  def name
    @data['name']
  end
  def version
    @data['version']
  end
  def uri
    @data['gem_uri']
  end
  def info
    @data['info']
  end
  def gem
    File.basename(uri)
  end
  def run(cmd)
    system(cmd)
    if $? && ($? >> 8) > 0
      raise GemWatch::Gem::CommandFailed, "[#{cmd} failed!]"
    end
  end
  def download(uri)
    open(uri, 'rb') do |downloaded_gem|
      File.open(gem, 'wb') do |file|
        file.write(downloaded_gem.read)
      end
    end
  end
  def download_and_convert!
    unless File.exist?(tarball_path)
      FileUtils.rm_rf(absolute_directory)
      FileUtils.mkdir_p(absolute_directory)
      Dir.chdir(absolute_directory) do
        download uri
        run "tar xfm #{gem}"
        run "tar xzfm data.tar.gz"
        run "zcat metadata.gz > metadata.yml"
        FileUtils.rm_f([gem, "data.tar.gz", "metadata.gz"])
      end
      Dir.chdir(File.dirname(absolute_directory)) do
        run "tar czf #{tarball} #{directory}"
        FileUtils.rm_rf(directory)
      end
    end
  end
  def directory
    "#{name}-#{version}"
  end
  def download_dir
    @download_dir ||= File.join(DOWNLOAD_BASE, name[0..0], name)
  end
  def absolute_directory
    File.join(download_dir, directory)
  end
  def tarball
    "#{directory}.tar.gz"
  end
  def tarball_uri
    "/#{File.basename(download_dir)}/#{tarball}"
  end
  def tarball_path
    File.join(download_dir, tarball)
  end
end
module HostHelper
  def host_with_port
    if request.respond_to?(:host_with_port)
      request.host_with_port
    else
      request.host + ([80,443].include?(request.port) ? '' : (':' + request.port.to_s))
    end
  end
{'fraction_non_alphanumeric': 0.12145174371451743, 'fraction_numerical': 0.0036496350364963502, 'mean_word_length': 2.9975688816855754, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3334358', 'n_tokens_mistral': 1715, 'n_tokens_neox': 1710, 'n_words': 378}