<commit_before>defmodule SGC.StargateCommand do
  alias SGC.StargateCommand.Response
  alias SGC.Cookie
  alias SGC.StargateCommand.Scraper.Profile

  def get(url, headers \\ [], cookies \\ nil) do
    request(:get, url, nil, headers, cookies)
  end

  def post(url, content, headers \\ [], cookies \\ nil) do
    request(:post, url, content, headers, cookies)
  end

  defp request(method, url, content, headers, cookies) do
    options = case cookies do
      nil -> []
      cookies ->
        cookies_formatted =
          cookies
          |> filter_cookies()
          |> Enum.map(&Cookie.format/1)
          |> Enum.join("; ")
        [hackney: [cookie: [cookies_formatted]]]
    end

    headers = [{"User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36"} | headers]

    %HTTPoison.Response{body: body, headers: headers, status_code: status_code} = case method do
      :get -> HTTPoison.get!(url, headers, options)
      :post -> HTTPoison.post!(url, content, headers, options)
    end

    response_cookies =
      headers
      |> Enum.filter(fn
        {"Set-Cookie", _} -> true
        _ -> false
      end)
      |> Enum.map(&Cookie.parse_from_header/1)

    %Response{
      status_code: status_code,
      body: body,
      headers: headers,
      cookies: response_cookies
    }
  end

  def filter_cookies(cookies) do
    cookies
    |> Enum.filter(&cookie_allowed?/1)
  end

  defp cookie_allowed?(%Cookie{name: name}) when name in ["_TopFan-BCK_session", "session_id", "guest_user_id"], do: true
  defp cookie_allowed?(_), do: false

  def login(username, password) do
    %Response{body: body, cookies: cookies} = get("https://www.stargatecommand.co/home")

    [{"meta", [{"name", "csrf-token"}, {"content", csrf_token}], []}] = Floki.find(body, "meta[name=csrf-token]")

    form = ["user[user_name]": username, "user[password]": password, "utf8": "✓", "authenticity_token": csrf_token]

    %Response{body: body, cookies: new_cookies, status_code: status_code} = post(
      "https://www.stargatecommand.co/login",
      {:form, form},
      [{"Accept", "application/json"}],
      cookies
    )

    new_cookies = Cookie.merge_lists(cookies, new_cookies)

    case status_code do
      200 -> {:ok, new_cookies}
      _ -> {:error, body}
    end
  end

  def user_info(cookies) do
    cookies = cookies |> filter_cookies()

    %Response{body: body, cookies: home_cookies} = get("https://www.stargatecommand.co/home", [], cookies)

    profile_url = Floki.find(body, ".user-level") |> Floki.attribute("href") |> List.first()

    case profile_url do
      "" -> {:error, :unauthenticated}
      url -> user_info(url, home_cookies)
    end
  end
  def user_info("/profile/" <> id, cookies) do
    cookies = cookies |> filter_cookies()

    %Response{status_code: status_code, body: body, cookies: profile_cookies} = get("https://www.stargatecommand.co/profile/#{id}", [], cookies)

    case status_code do
      200 ->
        data = %{
          user_id: id,
          username: Profile.username(body),
          profile_pic: Profile.profile_pic(body),
          header_pic: Profile.header_pic(body),
          all_access: Profile.all_access?(body),
          level: Profile.level(body),
          lifetime_command_coins: Profile.lifetime_command_coins(body),
          command_coins_available: Profile.command_coins_available(body),
          posts_count: Profile.posts_count(body),
          following_count: Profile.following_count(body),
          followers_count: Profile.followers_count(body)
        }

        new_cookies = Cookie.merge_lists(cookies, profile_cookies)

        {:ok, data, new_cookies}
      _ -> {:error, :unauthenticated}
    end
  end
  def user_info(id, cookies), do: user_info("/profile/#{id}", cookies)
end<commit_msg>handle 404 status on profile request<commit_after>defmodule SGC.StargateCommand do
  alias SGC.StargateCommand.Response
  alias SGC.Cookie
  alias SGC.StargateCommand.Scraper.Profile

  def get(url, headers \\ [], cookies \\ nil) do
    request(:get, url, nil, headers, cookies)
  end

  def post(url, content, headers \\ [], cookies \\ nil) do
    request(:post, url, content, headers, cookies)
  end

  defp request(method, url, content, headers, cookies) do
    options = case cookies do
      nil -> []
      cookies ->
        cookies_formatted =
          cookies
          |> filter_cookies()
          |> Enum.map(&Cookie.format/1)
          |> Enum.join("; ")
        [hackney: [cookie: [cookies_formatted]]]
    end

    headers = [{"User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36"} | headers]

    %HTTPoison.Response{body: body, headers: headers, status_code: status_code} = case method do
      :get -> HTTPoison.get!(url, headers, options)
      :post -> HTTPoison.post!(url, content, headers, options)
    end

    response_cookies =
      headers
      |> Enum.filter(fn
        {"Set-Cookie", _} -> true
        _ -> false
      end)
      |> Enum.map(&Cookie.parse_from_header/1)

    %Response{
      status_code: status_code,
      body: body,
      headers: headers,
      cookies: response_cookies
    }
  end

  def filter_cookies(cookies) do
    cookies
    |> Enum.filter(&cookie_allowed?/1)
  end

  defp cookie_allowed?(%Cookie{name: name}) when name in ["_TopFan-BCK_session", "session_id", "guest_user_id"], do: true
  defp cookie_allowed?(_), do: false

  def login(username, password) do
    %Response{body: body, cookies: cookies} = get("https://www.stargatecommand.co/home")

    [{"meta", [{"name", "csrf-token"}, {"content", csrf_token}], []}] = Floki.find(body, "meta[name=csrf-token]")

    form = ["user[user_name]": username, "user[password]": password, "utf8": "✓", "authenticity_token": csrf_token]

    %Response{body: body, cookies: new_cookies, status_code: status_code} = post(
      "https://www.stargatecommand.co/login",
      {:form, form},
      [{"Accept", "application/json"}],
      cookies
    )

    new_cookies = Cookie.merge_lists(cookies, new_cookies)

    case status_code do
      200 -> {:ok, new_cookies}
      _ -> {:error, body}
    end
  end

  def user_info(cookies) do
    cookies = cookies |> filter_cookies()

    %Response{body: body, cookies: home_cookies} = get("https://www.stargatecommand.co/home", [], cookies)

    profile_url = Floki.find(body, ".user-level") |> Floki.attribute("href") |> List.first()

    case profile_url do
      "" -> {:error, :unauthenticated}
      url -> user_info(url, home_cookies)
    end
  end
  def user_info("/profile/" <> id, cookies) do
    cookies = cookies |> filter_cookies()

    %Response{status_code: status_code, body: body, cookies: profile_cookies} = get("https://www.stargatecommand.co/profile/#{id}", [], cookies)

    case status_code do
      200 ->
        data = %{
          user_id: id,
          username: Profile.username(body),
          profile_pic: Profile.profile_pic(body),
          header_pic: Profile.header_pic(body),
          all_access: Profile.all_access?(body),
          level: Profile.level(body),
          lifetime_command_coins: Profile.lifetime_command_coins(body),
          command_coins_available: Profile.command_coins_available(body),
          posts_count: Profile.posts_count(body),
          following_count: Profile.following_count(body),
          followers_count: Profile.followers_count(body)
        }

        new_cookies = Cookie.merge_lists(cookies, profile_cookies)

        {:ok, data, new_cookies}
      404 -> {:error, :not_found}
      _ -> {:error, :unauthenticated}
    end
  end
  def user_info(id, cookies), do: user_info("/profile/#{id}", cookies)
end
{'fraction_non_alphanumeric': 0.14035764826965136, 'fraction_numerical': 0.010292036536729705, 'mean_word_length': 3.046850598646538, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 50, 'https://': 8, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6197792', 'n_tokens_mistral': 2622, 'n_tokens_neox': 2517, 'n_words': 543}