<commit_before>class UrlAvailabilityCheck

  def is_available(url, timeout = 3)
    begin
      response = Faraday.get do |req|
        req.url url
        req.options.timeout = timeout
        req.options.open_timeout = timeout
      end
      Rails.logger.debug("Status for url #{url} is #{response.status}")
      if (response.status == 200)
        return true
      end
      return false
    rescue
      Rails.logger.debug("Status for url #{url} cannot be checked")
      return false
    end
  end

end<commit_msg>Changed status code to handle<commit_after>class UrlAvailabilityCheck

  def is_available(url, timeout = 3)
    begin
      response = Faraday.get do |req|
        req.url url
        req.options.timeout = timeout
        req.options.open_timeout = timeout
      end
      Rails.logger.debug("Status for url #{url} is #{response.status}")
      if (response.status == 502 || response.status == 404)
        return false
      end
      return true
    rescue
      Rails.logger.debug("Status for url #{url} cannot be checked")
      return false
    end
  end

end
{'fraction_non_alphanumeric': 0.08747697974217311, 'fraction_numerical': 0.010128913443830571, 'mean_word_length': 2.3140243902439024, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5379288', 'n_tokens_mistral': 322, 'n_tokens_neox': 308, 'n_words': 97}