<commit_before>defmodule Glasnost.Orchestrator.AuthorSyncSup do
  use Supervisor
  alias Glasnost.Worker

  def start_link(arg \\ []) do
    Supervisor.start_link(__MODULE__, arg)
  end

  def init(arg) do
    children = :authors
      |> RuntimeConfig.get()
      |> AtomicMap.convert(safe: true)
      |> Enum.map(&validate_author_config/1)
      |> Enum.map(&put_in(&1,[:source_blockchain], RuntimeConfig.get(:source_blockchain)))
      |> Enum.map(&worker(Worker.AuthorSync, [&1], [id: "Wrk.AuthorSync:" <> &1.account_name]) )

    supervise(children, strategy: :one_for_one)
  end

  def validate_author_config(config) do
     # forcing a runtime pattern match exception if keys don't match
     # TODO: add value validation
     %{tags: %{blacklist: _, whitelist: _}, account_name: _ } = config
  end

end<commit_msg>delete all posts at the init supervisor stage to prevent stale on-disk mnesia caching<commit_after>defmodule Glasnost.Orchestrator.AuthorSyncSup do
  use Supervisor
  alias Glasnost.{Worker,Repo}

  def start_link(arg \\ []) do
    Supervisor.start_link(__MODULE__, arg)
  end

  def init(arg) do
    Repo.delete_all(from c in Glasnost.Post)
    children = :authors
      |> RuntimeConfig.get()
      |> AtomicMap.convert(safe: true)
      |> Enum.map(&validate_author_config/1)
      |> Enum.map(&put_in(&1,[:source_blockchain], RuntimeConfig.get(:source_blockchain)))
      |> Enum.map(&worker(Worker.AuthorSync, [&1], [id: "Wrk.AuthorSync:" <> &1.account_name]) )

    supervise(children, strategy: :one_for_one)
  end

  def validate_author_config(config) do
     # forcing a runtime pattern match exception if keys don't match
     # TODO: add value validation
     %{tags: %{blacklist: _, whitelist: _}, account_name: _ } = config
  end

end
{'fraction_non_alphanumeric': 0.12316384180790961, 'fraction_numerical': 0.004519774011299435, 'mean_word_length': 3.933147632311978, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5704343', 'n_tokens_mistral': 592, 'n_tokens_neox': 566, 'n_words': 130}