<commit_before># encoding: UTF-8
require "sinatra"
require "haml"
require "sass"
require "compass"
require "yaml"
require "yajl"
require 'digest/sha1'
require File.expand_path("../ext/www-middleware", __FILE__)

class HackerSays < Sinatra::Base
  configure do
    Compass.configuration do |config|
      config.project_path = public_folder
      config.sass_dir = "themes"
      config.images_dir = "themes"
      config.output_style = :compressed
    end

    set :haml, {:format => :html5}
    set :scss, Compass.sass_engine_options.
      merge(:views => File.join(public_folder, "themes"))
    use WwwMiddleware
  end

  def quotes
    return @quotes if production? && defined? @quotes

    begin
      f = File.open(File.dirname(__FILE__) + "/quotes.yaml", "r:utf-8")
      raw_quotes = YAML.load f.read
      @quotes = raw_quotes.reduce({}) do |memo, quote|
        quote[:c].gsub!($/, "<br>")
        quote[:a] = quote[:a] || "Anonymous"
        id = Digest::SHA1.hexdigest quote[:a] + quote[:c]
        memo[id[0..5]] = quote
        memo
      end
    ensure
      f.close
    end
  end

  def random_quotes
    ids = quotes.keys
    count = ids.size
    Array.new(10) {quote ids[rand count]}
  end

  def quote(id)
    return unless id
    quotes[id] && quotes[id].merge(:id => id)
  end<commit_msg>improve speed even more, but hash is still slow<commit_after># encoding: UTF-8
require "sinatra"
require "haml"
require "sass"
require "compass"
require "yaml"
require "yajl"
require 'digest/sha1'
require File.expand_path("../ext/www-middleware", __FILE__)

class HackerSays < Sinatra::Base
  configure do
    Compass.configuration do |config|
      config.project_path = public_folder
      config.sass_dir = "themes"
      config.images_dir = "themes"
      config.output_style = :compressed
    end

    set :haml, {:format => :html5}
    set :scss, Compass.sass_engine_options.
      merge(:views => File.join(public_folder, "themes"))
    use WwwMiddleware
  end

  def quotes
    return @quotes if production? && defined? @quotes

    begin
      f = File.open(File.dirname(__FILE__) + "/quotes.yaml", "r:utf-8")
      raw_quotes = YAML.load f.read
      @quotes = raw_quotes.reduce({}) do |memo, quote|
        quote[:c].gsub!($/, "<br>")
        quote[:a] = quote[:a] || "Anonymous"
        id = Digest::SHA1.hexdigest quote[:a] + quote[:c]
        memo[id[0..5]] = quote
        memo
      end
    ensure
      f.close
    end
  end

  def random_quotes
    ids = quotes.keys
    count = ids.size
    Array.new(10) {quote ids[rand count]}
  end

  def quote(id)
    return unless id
    quote = quotes[id]
    quote && quote.merge(:id => id)
  end
{'fraction_non_alphanumeric': 0.13433395872420262, 'fraction_numerical': 0.00675422138836773, 'mean_word_length': 2.8304597701149423, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1491101', 'n_tokens_mistral': 935, 'n_tokens_neox': 911, 'n_words': 225}