<commit_before>require "delegate"
require "sinatra"
require "csv"
require "json"
require "benchmark"
require File.expand_path("../../../lib/goldmine", __FILE__)

before do
  content_type "application/json"
end

get "/" do
  start = Time.now
  Benchmark.bm(15) do |x|
    x.report("raw") { raw }
    x.report("pivoted") { pivoted }
    x.report("computed") { computed }
    x.report("computed_tabular") { computed.to_tabular }
    x.report("computed_csv") { computed.to_tabular.to_csv }
  end

  JSON.dump(
    duration: Time.now - start,
    source_size: raw.size,
    computed_tabular: computed.to_tabular
  )
end

get "/raw" do
  JSON.dump(raw)
end

get "/pivoted" do
  JSON.dump(pivoted)
end

get "/computed" do
  JSON.dump(computed)
end

get "/computed_tabular" do
  JSON.dump(computed.to_tabular)
end

get "/computed_csv" do
  content_type "text/csv"
  computed.to_tabular.to_csv
end

private

def raw
  @raw ||= [].tap do |rows|
    file_path = File.expand_path("../DOITT_WIFI_HOTSPOT_01_13SEPT2010.csv", __FILE__)
    CSV.foreach(file_path, :headers => true) do |row|
      rows << row.to_hash
    end
  end
end

def pivoted
  @pivoted ||= Goldmine::ArrayMiner.new(raw)
    .pivot("City") { |row| row["CITY"] }
    .pivot("Zip Code") { |row| row["ZIP"] }
    .pivot("Area Code") { |row| row["PHONE"].to_s.gsub(/\W/, "")[0, 3] }
end

# NOTE: rollup blocks are called once for each pivot
#       best practice is to cache intermediate rollup results to avoid duplicate computations
#       leaving caching out for clarity
def computed
  @computed ||= begin
    value = pivoted.rollup("Total") { |list| list.size }

    value = value.rollup("Free") do |list|
      list.select { |row| !(row["TYPE"] =~ /free/i).nil? }.size
    end
<commit_msg>Clarify why we're not caching in the example<commit_after>require "delegate"
require "sinatra"
require "csv"
require "json"
require "benchmark"
require File.expand_path("../../../lib/goldmine", __FILE__)

before do
  content_type "application/json"
end

get "/" do
  start = Time.now
  Benchmark.bm(15) do |x|
    x.report("raw") { raw }
    x.report("pivoted") { pivoted }
    x.report("computed") { computed }
    x.report("computed_tabular") { computed.to_tabular }
    x.report("computed_csv") { computed.to_tabular.to_csv }
  end

  JSON.dump(
    duration: Time.now - start,
    source_size: raw.size,
    computed_tabular: computed.to_tabular
  )
end

get "/raw" do
  JSON.dump(raw)
end

get "/pivoted" do
  JSON.dump(pivoted)
end

get "/computed" do
  JSON.dump(computed)
end

get "/computed_tabular" do
  JSON.dump(computed.to_tabular)
end

get "/computed_csv" do
  content_type "text/csv"
  computed.to_tabular.to_csv
end

private

def raw
  @raw ||= [].tap do |rows|
    file_path = File.expand_path("../DOITT_WIFI_HOTSPOT_01_13SEPT2010.csv", __FILE__)
    CSV.foreach(file_path, :headers => true) do |row|
      rows << row.to_hash
    end
  end
end

def pivoted
  @pivoted ||= Goldmine::ArrayMiner.new(raw)
    .pivot("City") { |row| row["CITY"] }
    .pivot("Zip Code") { |row| row["ZIP"] }
    .pivot("Area Code") { |row| row["PHONE"].to_s.gsub(/\W/, "")[0, 3] }
end

# NOTE: rollup blocks are called once for each pivot
#       best practice is to cache intermediate rollup results to avoid duplicate computations
#       this example omits caching for simplicity & clarity
def computed
  @computed ||= begin
    value = pivoted.rollup("Total") { |list| list.size }

    value = value.rollup("Free") do |list|
      list.select { |row| !(row["TYPE"] =~ /free/i).nil? }.size
    end

{'fraction_non_alphanumeric': 0.14950634696755993, 'fraction_numerical': 0.006770098730606488, 'mean_word_length': 3.460377358490566, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5816210', 'n_tokens_mistral': 1327, 'n_tokens_neox': 1313, 'n_words': 317}