<commit_before>defmodule Nerves.Utils.HTTPClient do
  use GenServer

  @progress_steps 50

  def start_link() do
    {:ok, _} = Application.ensure_all_started(:nerves)
    start_httpc()
    GenServer.start_link(__MODULE__, [])
  end

  def stop(pid) do
    GenServer.stop(pid)
  end

  def get(_, _, _ \\ [])

  def get(_pid, %URI{host: nil, path: path}, _opts) do
    path
    |> Path.expand()
    |> File.read()
  end

  def get(pid, %URI{} = uri, opts) do
    url = URI.to_string(uri)
    get(pid, url, opts)
  end

  def get(pid, url, opts), do: GenServer.call(pid, {:get, url, opts}, :infinity)

  def init([]) do
    {:ok,
     %{
       url: nil,
       content_length: 0,
       buffer: "",
       buffer_size: 0,
       filename: "",
       caller: nil,
       number_of_redirects: 0,
       progress?: true,
       get_opts: []
     }}
  end

  def handle_call({:get, _url, _opts}, _from, %{number_of_redirects: n} = s) when n > 5 do
    GenServer.reply(s.caller, {:error, :too_many_redirects})
    {:noreply, %{s | url: nil, number_of_redirects: 0, caller: nil}}
  end

  def handle_call({:get, url, opts}, from, s) do
    progress? = Keyword.get(opts, :progress?, true)

    user_headers = Keyword.get(opts, :headers, []) |> Enum.map(&tuple_to_charlist/1)

    headers = [
      {'User-Agent', 'Nerves/#{Nerves.version()}'},
      {'Content-Type', 'application/octet-stream'} | user_headers
    ]

    http_opts =
      [
        timeout: :infinity,
        autoredirect: false,
        ssl: [
          verify: :verify_peer,
          cacertfile: CAStore.file_path(),
          customize_hostname_check: [
            {:match_fun, :public_key.pkix_verify_hostname_match_fun(:https)}
          ]
        ]
      ]
      |> Keyword.merge(Nerves.Utils.Proxy.config(url))
      |> Keyword.merge(Keyword.get(opts, :http_opts, []))

    :httpc.request(
      :get,
      {String.to_charlist(url), headers},
      http_opts,
      [stream: :self, receiver: self(), sync: false],
      :nerves
    )

    {:noreply, %{s | url: url, caller: from, get_opts: opts, progress?: progress?}}
  end
<commit_msg>add ssl depth 3 to http_opts<commit_after>defmodule Nerves.Utils.HTTPClient do
  use GenServer

  @progress_steps 50

  def start_link() do
    {:ok, _} = Application.ensure_all_started(:nerves)
    start_httpc()
    GenServer.start_link(__MODULE__, [])
  end

  def stop(pid) do
    GenServer.stop(pid)
  end

  def get(_, _, _ \\ [])

  def get(_pid, %URI{host: nil, path: path}, _opts) do
    path
    |> Path.expand()
    |> File.read()
  end

  def get(pid, %URI{} = uri, opts) do
    url = URI.to_string(uri)
    get(pid, url, opts)
  end

  def get(pid, url, opts), do: GenServer.call(pid, {:get, url, opts}, :infinity)

  def init([]) do
    {:ok,
     %{
       url: nil,
       content_length: 0,
       buffer: "",
       buffer_size: 0,
       filename: "",
       caller: nil,
       number_of_redirects: 0,
       progress?: true,
       get_opts: []
     }}
  end

  def handle_call({:get, _url, _opts}, _from, %{number_of_redirects: n} = s) when n > 5 do
    GenServer.reply(s.caller, {:error, :too_many_redirects})
    {:noreply, %{s | url: nil, number_of_redirects: 0, caller: nil}}
  end

  def handle_call({:get, url, opts}, from, s) do
    progress? = Keyword.get(opts, :progress?, true)

    user_headers = Keyword.get(opts, :headers, []) |> Enum.map(&tuple_to_charlist/1)

    headers = [
      {'User-Agent', 'Nerves/#{Nerves.version()}'},
      {'Content-Type', 'application/octet-stream'} | user_headers
    ]

    http_opts =
      [
        timeout: :infinity,
        autoredirect: false,
        ssl: [
          verify: :verify_peer,
          cacertfile: CAStore.file_path(),
          depth: 3,
          customize_hostname_check: [
            {:match_fun, :public_key.pkix_verify_hostname_match_fun(:https)}
          ]
        ]
      ]
      |> Keyword.merge(Nerves.Utils.Proxy.config(url))
      |> Keyword.merge(Keyword.get(opts, :http_opts, []))

    :httpc.request(
      :get,
      {String.to_charlist(url), headers},
      http_opts,
      [stream: :self, receiver: self(), sync: false],
      :nerves
    )

    {:noreply, %{s | url: url, caller: from, get_opts: opts, progress?: progress?}}
  end

{'fraction_non_alphanumeric': 0.15337567631145613, 'fraction_numerical': 0.004234297812279464, 'mean_word_length': 2.573109243697479, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1662269', 'n_tokens_mistral': 1575, 'n_tokens_neox': 1502, 'n_words': 327}