<commit_before>defmodule FourOhFourFinderApp.PageAnalyzer do
  use HTTPoison.Base
  require Logger

  def analyze(url) do
    case get_with_redirects(url) do
      {:ok, %HTTPoison.Response{status_code: code, body: body}} ->
        html = Floki.parse(body)
        if good_resp?(code) do
          %{
            url: url,
            http_status: code,
            success: true,
            outgoing_hrefs: tree_hrefs(html, url),
            ids: tree_ids(html)
          }
        else
          %{ url: url, http_status: code, success: false }
        end

      {:error, %HTTPoison.Error{reason: reason}} ->
        %{
          url: url,
          error: reason,
          success: false
        }
    end
  end


  def get_with_redirects(url) do
    case get(url) do
      {:ok, response} ->
        if redirect?(response.status_code) do
          Logger.debug("will redirect " <> redirect_location(response.headers))
          get_with_redirects(resolve_url(redirect_location(response.headers), url))
        else
          {:ok, response}
        end

      {:error, response} ->
        {:error, response}
    end
  end

  def redirect_location(headers) do
    elem(Enum.find(headers, fn({header, _}) -> "location" == String.downcase(header) end), 1)
  end

  def good_resp?(code) do<commit_msg>check response content-type<commit_after>defmodule FourOhFourFinderApp.PageAnalyzer do
  use HTTPoison.Base

  def analyze(url) do
    case get_with_redirects(url) do
      {:ok, response = %HTTPoison.Response{status_code: code, body: body}} ->
        if good_resp?(code) do
          if is_html(response) do
            html = Floki.parse(body)
            %{
              url: url,
              http_status: code,
              success: true,
              outgoing_hrefs: tree_hrefs(html, url),
              ids: tree_ids(html)
            }
          else
            %{
              url: url,
              http_status: code,
              success: true,
              outgoing_hrefs: [],
              ids: []
            }
          end
        else
          %{ url: url, http_status: code, success: false }
        end

      {:error, %HTTPoison.Error{reason: reason}} ->
        %{
          url: url,
          error: reason,
          success: false
        }
    end
  end


  def get_with_redirects(url) do
    case get(url) do
      {:ok, response} ->
        if redirect?(response.status_code) do
          get_with_redirects(resolve_url(get_header(response.headers, "Location"), url))
        else
          {:ok, response}
        end

      {:error, response} ->
        {:error, response}
    end
  end

  def is_html(response) do
    String.starts_with?(get_header(response.headers, "content-type"), "text/html")
  end

  def get_header(headers, header_name) do
    elem(Enum.find(headers, fn({header, _}) -> String.downcase(header_name) == String.downcase(header) end), 1)
  end

  def good_resp?(code) do
{'fraction_non_alphanumeric': 0.10959836623553437, 'fraction_numerical': 0.0006807351940095302, 'mean_word_length': 1.9070227497527201, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '336543', 'n_tokens_mistral': 913, 'n_tokens_neox': 907, 'n_words': 203}