<commit_before>defmodule Glasnost.Worker.AuthorSync do
  use GenServer
  alias Glasnost.Repo
  import Logger

  def start_link(args, opts \\ []) do
    GenServer.start_link(__MODULE__, args, opts)
  end

  def init(config) do
    import Ecto.Query
    %{account_name: account_name} = config
    Logger.info("AuthorSync GenServer process for #{account_name} is being initialized...")
    Repo.delete_all(from c in Glasnost.Post, where: c.author == ^account_name)

    config = config
      |> put_in([:client_mod], RuntimeConfig.blockchain_client_mod)
      |> put_in([:current_cursor], "")
    Process.send_after(self(), :tick, 1_000)
    {:ok, config}
  end

  def handle_info(:tick, state) do
     utc_now_str = NaiveDateTime.utc_now |> NaiveDateTime.to_iso8601 |> trim_trailing_ms
     %{account_name: account_name, current_cursor: current_cursor,
      client_mod: client_mod, filters: filters} = state
     {:ok, posts} = client_mod.get_discussions_by_author_before_date(account_name, current_cursor, utc_now_str, 100)
     posts = posts
      |> Enum.map(&parse_json_metadata/1)
      |> Enum.map(&extract_put_tags/1)
      |> filter_whitelisted(filters.tags.whitelist)
      |> filter_blacklisted(filters.tags.blacklist)
      |> filter_by_title(filters.title)
      |> List.flatten

     for post <- posts do
       save_to_db(post)
     end
     state = iterate(posts, state)
     {:noreply, state}
  end

  def filter_by_title(posts, %{whitelist: [], blacklist: []}) do
    posts
  end

  def filter_by_title(posts, %{whitelist: whls, blacklist: blls}) do
    regexes_compiler = &Enum.reduce(&1, [], fn regex_str, acc ->
      case Regex.compile(regex_str) do
         {:ok, regex} -> acc ++ [regex]
         {:error, _} -> acc
      end
    end)
    whls_regexes = regexes_compiler.(whls)
    blls_regexes = regexes_compiler.(blls)

    matches_any_wl_regex? = fn title ->
      case whls_regexes do
        [] -> true
        _ -> Enum.reduce(whls_regexes, false, fn regex, acc ->
        acc or String.match?(title, regex)
      end)
      end<commit_msg>add filter by created fn<commit_after>defmodule Glasnost.Worker.AuthorSync do
  use GenServer
  alias Glasnost.Repo
  import Logger

  def start_link(args, opts \\ []) do
    GenServer.start_link(__MODULE__, args, opts)
  end

  def init(config) do
    import Ecto.Query
    %{account_name: account_name} = config
    Logger.info("AuthorSync GenServer process for #{account_name} is being initialized...")
    Repo.delete_all(from c in Glasnost.Post, where: c.author == ^account_name)

    config = config
      |> put_in([:client_mod], RuntimeConfig.blockchain_client_mod)
      |> put_in([:current_cursor], "")
    Process.send_after(self(), :tick, 1_000)
    {:ok, config}
  end

  def handle_info(:tick, state) do
     utc_now_str = NaiveDateTime.utc_now |> NaiveDateTime.to_iso8601 |> trim_trailing_ms
     %{account_name: account_name, current_cursor: current_cursor,
      client_mod: client_mod, filters: filters} = state
     {:ok, posts} = client_mod.get_discussions_by_author_before_date(account_name, current_cursor, utc_now_str, 100)
     posts = posts
      |> Enum.map(&parse_json_metadata/1)
      |> Enum.map(&extract_put_tags/1)
      |> filter_whitelisted(filters.tags.whitelist)
      |> filter_blacklisted(filters.tags.blacklist)
      |> filter_by_title(filters.title)
      |> List.flatten
      |> filter_by_created(filters.created)

     for post <- posts do
       save_to_db(post)
     end
     state = iterate(posts, state)
     {:noreply, state}
  end

  def filter_by_created(posts, %{only_after: "", only_before: ""}) do
    posts
  end

  def filter_by_created(posts, %{only_after: only_after, only_before: only_before}) do
    # offset is always zero, but required by ISO standard
    only_after = if only_after == "", do:  "1970-01-01", else: only_after
    only_before = if only_before == "", do:  "2038-01-01", else: only_before
    {:ok, only_after, 0} = DateTime.from_iso8601(only_after <> "T00:00:00Z")
    {:ok, only_before, 0} = DateTime.from_iso8601(only_before <> "T00:00:00Z")
    for post <- posts,
    {:ok, created, 0} = DateTime.from_iso8601(post["created"] <> "Z"),
      DateTime.compare(created, only_after) === :gt,
      DateTime.compare(created, only_before) === :lt,
      do: post
  end

  def filter_by_title(posts, %{whitelist: [], blacklist: []}) do
    posts
  end

  def filter_by_title(posts, %{whitelist: whls, blacklist: blls}) do
    regexes_compiler = &Enum.reduce(&1, [], fn regex_str, acc ->
      case Regex.compile(regex_str) do
         {:ok, regex} -> acc ++ [regex]
         {:error, _} -> acc
      end
    end)
    whls_regexes = regexes_compiler.(whls)
    blls_regexes = regexes_compiler.(blls)

    matches_any_wl_regex? = fn title ->
      case whls_regexes do
        [] -> true
        _ -> Enum.reduce(whls_regexes, false, fn regex, acc ->
        acc or String.match?(title, regex)
      end)
      end
{'fraction_non_alphanumeric': 0.12376337573187966, 'fraction_numerical': 0.014334746618211184, 'mean_word_length': 3.3228621291448515, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '965532', 'n_tokens_mistral': 1768, 'n_tokens_neox': 1682, 'n_words': 360}