<commit_before>require 'sinatra'
set :server, :thin
#set :port, 80
set :public_folder, File.dirname(__FILE__) + '/public'


require 'pp'

get '/text/*.txt' do  | file |
  path = File.dirname(__FILE__) +"/text/#{file}.txt"
  IO.readlines(path)
end
get '/search.json/:query' do
  content_type 'application/json'
  results = []
  query = params[:query].to_s
  Dir.glob(File.dirname(__FILE__) + "/text/*.txt").each_with_index { |f,i| 
    str = IO.readlines(f).to_s
    page =  /[0-9].+/.match(f).to_s.split("_").last().split(".").first()
    if(/#{query}/i.match(str))
      results << page.to_i
    end
  }
  "{\"results\" : #{results},\"query\": \"#{query}\"}"
end

get '/cfg.json' do
  content_type 'application/x-javascript'
  erb :config
end
get '/' do
  erb :index 
end
<commit_msg>search by word<commit_after>require 'sinatra'
set :server, :thin
#set :port, 80
set :public_folder, File.dirname(__FILE__) + '/public'


require 'pp'

get '/text/*.txt' do  | file |
  path = File.dirname(__FILE__) +"/text/#{file}.txt"
  IO.readlines(path)
end
get '/search.json/:query' do
  content_type 'application/json'
  results = []
  query = params[:query].to_s
  Dir.glob(File.dirname(__FILE__) + "/text/*.txt").each_with_index { |f,i| 
    str = IO.readlines(f).to_s
    page =  /[0-9].+/.match(f).to_s.split("_").last().split(".").first()
    words = str.split(" ")
    if (words.include? query)
    #if(/#{query}/i.match(str))
      results << page.to_i
    end
  }
  "{\"results\" : #{results},\"query\": \"#{query}\"}"
end

get '/cfg.json' do
  content_type 'application/x-javascript'
  erb :config
end
get '/' do
  erb :index 
end

{'fraction_non_alphanumeric': 0.21178637200736647, 'fraction_numerical': 0.004910988336402701, 'mean_word_length': 3.4054054054054053, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '266633', 'n_tokens_mistral': 637, 'n_tokens_neox': 626, 'n_words': 125}