<commit_before>require 'open-uri'
require 'sinatra/base'
require 'json'
require 'ostruct'

class Array
  
  def to_h
    reduce({}) { |r, e| r[e[0]] = e[1]; r }
  end
  
end

class Object

  def map(&f)
    f.(self)
  end
  
end

class SearchPhraseWebApp < Sinatra::Application
  
  get '/:board/res/:thread.html' do
    #
    headers "Content-Type" => "text/html; charset=utf8"
    # 
    uri = "https://2ch.hk/makaba/mobile.fcgi?task=get_thread&board=#{params[:board]}&thread=#{params[:thread]}&post=0"
    #
    posts =
      begin
        open(uri, HEADERS) { |io| io.read }.
          map { |s| JSON.parse("{\"data\": #{s}}")['data'] }.
          tap { |r| halt 404, r["Error"] if r.is_a? Hash and r.key? "Error" }.
          map do |post|
            post = OpenStruct.new(post)
            post.files ||= []
            post.files.map! { |file| OpenStruct.new(file) }
            post
          end.
          each_with_index { |post, i| post.rel_num = i+1 }
      rescue OpenURI::HTTPError => e<commit_msg>Use new API and fix bugs.<commit_after>require 'open-uri'
require 'sinatra/base'
require 'json'
require 'ostruct'

class Array
  
  def to_h
    reduce({}) { |r, e| r[e[0]] = e[1]; r }
  end
  
end

class Object
  
  def map1(&f)
    f.(self)
  end
  
end

class SearchPhraseWebApp < Sinatra::Application
  
  get '/:board/res/:thread.html' do
    #
    headers "Content-Type" => "text/html; charset=utf8"
    # 
    uri = "https://2ch.hk/#{params[:board]}/res/#{params[:thread]}.json"
    #
    posts =
      begin
        open(uri, HEADERS) { |io| io.read }.
          map1 { |s| JSON.parse("{\"data\": #{s}}")['data'] }.
          tap { |r| halt 404, r["Error"] if r.is_a? Hash and r.key? "Error" }.
          map1 { |d| d['threads'][0]['posts'] }.
          map do |post|
            post = OpenStruct.new(post)
            post.files ||= []
            post.files.map! { |file| OpenStruct.new(file) }
            post
          end.
          each_with_index { |post, i| post.rel_num = i+1 }
      rescue OpenURI::HTTPError => e
{'fraction_non_alphanumeric': 0.17537860283341475, 'fraction_numerical': 0.010258915486077186, 'mean_word_length': 2.1604938271604937, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2033983', 'n_tokens_mistral': 768, 'n_tokens_neox': 758, 'n_words': 165}