<commit_before>require 'sinatra'
require 'json'
require 'pp'

get '/' do
  'This is a web hook!'
end

post '/' do
  payload = JSON.parse(params[:payload])
  #pp payload
  handle_payload(payload)
end

def handle_payload(payload)
  events = payload.delete('events') or raise 'No events!'
  pp payload
  puts "Got #{events.size} events!"
  events.group_by do |event|
    message = event['message']
    if message
      message[/^Error (\w+)/, 1]
    else
      puts 'WARNING: event has no message!'
      nil
    end
  end.each do |error, events|
    puts "Error #{error}: #{events.size} events."
  end

  'OK'
rescue => e
  [500, "#{e.class}: #{e}"]
end<commit_msg>Cleanup<commit_after>require 'sinatra'
require 'json'
require 'pp'

get '/' do
  'This is a web hook!'
end

post '/' do
  payload = JSON.parse(params[:payload])
  begin
    handle_payload(payload)
    'OK'
  rescue => e
    [500, "#{e.class}: #{e}"]
  end
end

def handle_payload(payload)
  events = payload.delete('events') or raise 'No events!'
  pp payload
  puts "Got #{events.size} events!"
  events.group_by do |event|
    message = event['message']
    if message
      message[/^Error (\w+)/, 1]
    else
      puts 'WARNING: event has no message!'
      nil
    end
  end.each do |error, events|
    puts "Error #{error}: #{events.size} events."
  end
end
{'fraction_non_alphanumeric': 0.14996232102486812, 'fraction_numerical': 0.006028636021100226, 'mean_word_length': 3.124223602484472, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4495038', 'n_tokens_mistral': 466, 'n_tokens_neox': 461, 'n_words': 143}