<commit_before>require 'resolv'

class HomeController < ApplicationController
  def index
    unless params[:page].nil?
      @page_num = params[:page]   
      unless (1..3).include? @page_num.to_i
        not_found
      end
    end

    return forbidden if Bot.find_by(ip: request.remote_ip) && 
        ENV['MISBEHAVE_ADMIN_IP'] != request.remote_ip

    b = Bot.new
    b.ip         = request.remote_ip
    b.user_agent = request.headers["User-Agent"]
    b.hostname   = nil

    b.is_crawler = true if crawler?(b.user_agent)
    b.save

    # if b.valid?    
    #   b.save
    # else
    #   b = nil
    #   forbidden unless ENV['MISBEHAVE_ADMIN_IP'] == request.remote_ip
    # end

  end<commit_msg>set default hostname of a bot to empty string<commit_after>require 'resolv'

class HomeController < ApplicationController
  def index
    unless params[:page].nil?
      @page_num = params[:page]   
      unless (1..3).include? @page_num.to_i
        not_found
      end
    end

    return forbidden if Bot.find_by(ip: request.remote_ip) && 
        ENV['MISBEHAVE_ADMIN_IP'] != request.remote_ip

    b = Bot.new
    b.ip         = request.remote_ip
    b.user_agent = request.headers["User-Agent"]
    b.hostname   = ""

    b.is_crawler = true if crawler?(b.user_agent)
    b.save

    # if b.valid?    
    #   b.save
    # else
    #   b = nil
    #   forbidden unless ENV['MISBEHAVE_ADMIN_IP'] == request.remote_ip
    # end

  end
{'fraction_non_alphanumeric': 0.10795847750865052, 'fraction_numerical': 0.002768166089965398, 'mean_word_length': 2.308924485125858, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1537410', 'n_tokens_mistral': 518, 'n_tokens_neox': 505, 'n_words': 104}