<commit_before>defmodule FarmbotExt.API.DirtyWorker do
  @moduledoc "Handles uploading/downloading of data from the API."
  alias FarmbotCore.Asset.{Private, Repo}

  alias FarmbotExt.{API, API.DirtyWorker}
  import API.View, only: [render: 2]

  require Logger
  require FarmbotCore.Logger
  use GenServer
  @timeout 500
  # these resources can't be accessed by `id`.
  @singular [
    FarmbotCore.Asset.Device,
    FarmbotCore.Asset.FirmwareConfig,
    FarmbotCore.Asset.FbosConfig
  ]

  @doc false
  def child_spec(module) when is_atom(module) do
    %{
      id: {DirtyWorker, module},
      start: {__MODULE__, :start_link, [[module: module, timeout: @timeout]]},
      type: :worker,
      restart: :permanent,
      shutdown: 500
    }
  end

  @doc "Start an instance of a DirtyWorker"
  def start_link(args) do
    GenServer.start_link(__MODULE__, args)
  end

  @impl GenServer
  def init(args) do
    module = Keyword.fetch!(args, :module)
    Process.send_after(self(), :do_work, @timeout)
    {:ok, %{module: module}}
  end

  @impl GenServer
  def handle_info(:do_work, %{module: module} = state) do
    Process.sleep(@timeout)
    maybe_resync()
    maybe_upload(module)
    Process.send_after(self(), :do_work, @timeout)
    {:noreply, state}
  end

  def work(dirty, module) do
    # Go easy on the API
    Process.sleep(@timeout)
    response = http_request(dirty, module)
    handle_http_response(dirty, module, response)
  end

  defp http_request(%{id: nil} = dirty, module) do
    path = module.path()
    data = render(module, dirty)
    API.post(API.client(), path, data)
  end

  defp http_request(dirty, module) when module in @singular do
    path = path = module.path()
    data = render(module, dirty)
    API.patch(API.client(), path, data)
  end

  defp http_request(dirty, module) do
    path = Path.join(module.path(), to_string(dirty.id))
    data = render(module, dirty)
    API.patch(API.client(), path, data)
  end

  # This is a fix for a race condtion. The root cause is unknown
  # as of 18 May 2020. The problem is that records are marked
  # dirty _before_ the dirty data is saved. That means that FBOS
  # knows a record has changed, but for a brief moment, it only
  # has the old copy of the record (not the changes).
  # Because of this race condtion,
  # The race condition:
  #
  # * Is nondeterministic
  # * Happens frequently when running many MARK AS steps in one go.
  # * Happens frequently when Erlang VM only has one thread
  #    * Ie: `iex --erl '+S 1 +A 1' -S mix`
  # * Happens frequently when @timeout is decreased to `1`.
  #
  # This function PREVENTS CORRUPTION OF API DATA. It can be
  # removed once the root cause of the data race is determined.
  #   - RC 18 May 2020
  def race_free?(module, list) do
    Enum.find(list, fn item ->
      if item.id do
        if item == Repo.get_by(module, id: item.id) do
          # This item is OK - no race condition.
          false
        else
          # There was a race condtion. We probably can't trust
          # any of the data in this list. We need to wait and
          # try again later.
          Process.sleep(@timeout * 3)
          true
        end
      else
        # This item only exists on the FBOS side.
        # It will never be affected by the data race condtion.
        false
      end
    end)
  end

  def maybe_resync(timeout \\ @timeout) do
    if Private.any_stale?() do
      FarmbotCore.Logger.error(4, "Stale data detected. This may be related to a slow internet connection.")
      Private.recover_from_row_lock_failure()
      Process.sleep(timeout * 2)
      FarmbotCeleryScript.SysCalls.sync()
      Process.sleep(timeout * 8)
      true
    else
      false
    end
  end

  def maybe_upload(module) do
    list = Enum.uniq(Private.list_dirty(module) ++ Private.list_local(module))

    if race_free?(module, list) do
      Enum.map(list, fn dirty -> work(dirty, module) end)
    end
  end

  # Valid data
  def handle_http_response(dirty, module, {:ok, %{status: s, body: body}})
      when s > 199 and
             s < 300 do
    dirty |> module.changeset(body) |> finalize(module)
  end

  def handle_http_response(dirty, module, {:ok, %{status: s}}) when s == 409 do
    FarmbotCore.Logger.error(2, "Stale data detected. Sync required.")
    Private.mark_stale!(module.changeset(dirty))
  end

  # Invalid data
  def handle_http_response(dirty, module, {:ok, %{status: s, body: %{} = body}})
      when s > 399 and
             s < 500 do
    FarmbotCore.Logger.error(2, "HTTP Error #{s}. #{inspect(body)}")
    changeset = module.changeset(dirty)

    Enum.reduce(body, changeset, fn {key, val}, changeset ->
      Ecto.Changeset.add_error(changeset, key, val)
    end)
    |> finalize(module)
  end

  # Invalid data, but the API didn't say why
  def handle_http_response(dirty, module, {:ok, %{status: s, body: _body}})<commit_msg>Revert changes (Thanks JSimmonds)<commit_after>defmodule FarmbotExt.API.DirtyWorker do
  @moduledoc "Handles uploading/downloading of data from the API."
  alias FarmbotCore.Asset.{Private, Repo}

  alias FarmbotExt.{API, API.DirtyWorker}
  import API.View, only: [render: 2]

  require Logger
  require FarmbotCore.Logger
  use GenServer
  @timeout 500
  # these resources can't be accessed by `id`.
  @singular [
    FarmbotCore.Asset.Device,
    FarmbotCore.Asset.FirmwareConfig,
    FarmbotCore.Asset.FbosConfig
  ]

  @doc false
  def child_spec(module) when is_atom(module) do
    %{
      id: {DirtyWorker, module},
      start: {__MODULE__, :start_link, [[module: module, timeout: @timeout]]},
      type: :worker,
      restart: :permanent,
      shutdown: 500
    }
  end

  @doc "Start an instance of a DirtyWorker"
  def start_link(args) do
    GenServer.start_link(__MODULE__, args)
  end

  @impl GenServer
  def init(args) do
    module = Keyword.fetch!(args, :module)
    Process.send_after(self(), :do_work, @timeout)
    {:ok, %{module: module}}
  end

  @impl GenServer
  def handle_info(:do_work, %{module: module} = state) do
    Process.sleep(@timeout)
    maybe_resync()
    maybe_upload(module)
    Process.send_after(self(), :do_work, @timeout)
    {:noreply, state}
  end

  def work(dirty, module) do
    # Go easy on the API
    Process.sleep(@timeout)
    response = http_request(dirty, module)
    handle_http_response(dirty, module, response)
  end

  defp http_request(%{id: nil} = dirty, module) do
    path = module.path()
    data = render(module, dirty)
    API.post(API.client(), path, data)
  end

  defp http_request(dirty, module) when module in @singular do
    path = path = module.path()
    data = render(module, dirty)
    API.patch(API.client(), path, data)
  end

  defp http_request(dirty, module) do
    path = Path.join(module.path(), to_string(dirty.id))
    data = render(module, dirty)
    API.patch(API.client(), path, data)
  end

  # This is a fix for a race condtion. The root cause is unknown
  # as of 18 May 2020. The problem is that records are marked
  # dirty _before_ the dirty data is saved. That means that FBOS
  # knows a record has changed, but for a brief moment, it only
  # has the old copy of the record (not the changes).
  # Because of this race condtion,
  # The race condition:
  #
  # * Is nondeterministic
  # * Happens frequently when running many MARK AS steps in one go.
  # * Happens frequently when Erlang VM only has one thread
  #    * Ie: `iex --erl '+S 1 +A 1' -S mix`
  # * Happens frequently when @timeout is decreased to `1`.
  #
  # This function PREVENTS CORRUPTION OF API DATA. It can be
  # removed once the root cause of the data race is determined.
  #   - RC 18 May 2020
  def has_race_condition?(module, list) do
    Enum.find(list, fn item ->
      if item.id do
        if item == Repo.get_by(module, id: item.id) do
          # This item is OK - no race condition.
          true
        else
          # There was a race condtion. We probably can't trust
          # any of the data in this list. We need to wait and
          # try again later.
          Process.sleep(@timeout * 3)
          true
        end
      else
        # This item only exists on the FBOS side.
        # It will never be affected by the data race
        # condtion.
        true
      end
    end)
  end

  def maybe_resync(timeout \\ @timeout) do
    if Private.any_stale?() do
      FarmbotCore.Logger.error(
        4,
        "Stale data detected. Please check internet onnection and re-sync."
      )

      Private.recover_from_row_lock_failure()
      Process.sleep(timeout * 2)
      FarmbotCeleryScript.SysCalls.sync()
      Process.sleep(timeout * 8)
      true
    else
      false
    end
  end

  def maybe_upload(module) do
    list = Enum.uniq(Private.list_dirty(module) ++ Private.list_local(module))

    unless has_race_condition?(module, list) do
      Enum.map(list, fn dirty -> work(dirty, module) end)
    end
  end

  # Valid data
  def handle_http_response(dirty, module, {:ok, %{status: s, body: body}})
      when s > 199 and
             s < 300 do
    dirty |> module.changeset(body) |> finalize(module)
  end

  def handle_http_response(dirty, module, {:ok, %{status: s}}) when s == 409 do
    FarmbotCore.Logger.error(2, "Stale data detected. Sync required.")
    Private.mark_stale!(module.changeset(dirty))
  end

  # Invalid data
  def handle_http_response(dirty, module, {:ok, %{status: s, body: %{} = body}})
      when s > 399 and
             s < 500 do
    FarmbotCore.Logger.error(2, "HTTP Error #{s}. #{inspect(body)}")
    changeset = module.changeset(dirty)

    Enum.reduce(body, changeset, fn {key, val}, changeset ->
      Ecto.Changeset.add_error(changeset, key, val)
    end)
    |> finalize(module)
  end

  # Invalid data, but the API didn't say why
  def handle_http_response(dirty, module, {:ok, %{status: s, body: _body}})
{'fraction_non_alphanumeric': 0.10525246367977242, 'fraction_numerical': 0.008737173625927054, 'mean_word_length': 2.842310694769711, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2359823', 'n_tokens_mistral': 3347, 'n_tokens_neox': 3144, 'n_words': 1040}