<commit_before>defmodule Pullhub.PullRequestsFetcher do
  require Logger

  use GenServer


  import Ecto.Query


  alias Pullhub.Repo
  alias Pullhub.User
  alias Pullhub.GithubApi
  alias Pullhub.Repository
  alias Pullhub.PullRequest
  alias Pullhub.RepositoriesService

  def start_link(opts \\ []) do
    {:ok, pid} = GenServer.start_link(__MODULE__, [], opts)
  end

  def fetch(fetch_params) do
    GenServer.cast(:pull_requests_fetcher, {:fetch, fetch_params})
  end

  def handle_cast({:fetch, fetch_params}, state) do
    {:noreply, fetch}
  end

  def init([]) do
    schedule_work()
    {:ok, []}
  end

  def fetch do
    Repo.all(User)
    |> Enum.filter(fn(u) -> u.github_token != nil end)
    |> Enum.map(&fetch_and_update_pull_requests/1)
  end

  def fetch_and_update_pull_requests(user) do
    client = Tentacat.Client.new(%{access_token: user.github_token})
    user.id
    |> Repository.user_repositories
    |> Repository.enabled_repositories
    |> Repo.all
    |> update_pull_requests(user, client)
  end

  def update_pull_requests(repositories, user, client) do
    Enum.map(repositories, fn(repo) ->
      Pullhub.GithubApi.pull_requests(repo, user)
      |> Enum.map(&extract_pull_info/1)
      |> upsert_pulls(repo)
    end)
  end

  defp upsert_pulls(pulls, repo) do
    Enum.map(pulls, fn(pull) ->
      query = from p in PullRequest, where: p.remote_id == ^pull.remote_id and p.repository_id == ^repo.id

      pull = Map.put(pull, :repository_id, repo.id)

      if !Repo.one(query) do
        Repo.insert(PullRequest.changeset(%PullRequest{}, pull))
      else
        Repo.update(PullRequest.changeset(Repo.one(query), pull))
      end
    end)
  end

  defp extract_pull_info(pull) do
    %{
      assignees_logins: Enum.map(pull["assignees"], fn(user) -> user["login"] end),
      body: pull["body"],
      issue_url: pull["issue_url"],
      remote_id: pull["id"],
      repository_id: nil,
      state: pull["state"],
      title: pull["title"],
      url: pull["html_url"],
      author_login: pull["user"]["login"]
    }
  end

  def convert_repository_to_structs(repos, user) do
    Enum.map(repos, fn(repo) -> %{
      name: repo.name,
      owner: repo.owner,
      remote_id: repo.remote_id,
      user_id: user.id
    } end)
  end

  def handle_info(:fetch, state) do
    state = fetch(%{})
    schedule_work()
    {:noreply, state}
  end

  defp schedule_work do
    Process.send_after(self(), :fetch, 120_000)
  end
end
<commit_msg>Cleanup PullRequestsFetcher and add new store strategy<commit_after>defmodule Pullhub.PullRequestsFetcher do
  use GenServer

  alias Pullhub.User
  alias Pullhub.PullRequestsBroadcaster
  alias Pullhub.PullRequestsSync

  def fetch(fetch_params) do
    GenServer.cast(:pull_requests_fetcher,
                  {:fetch_user_pull_requests, fetch_params})
  end

  def fetch_and_update_pull_requests(%User{} = user) do
    user
    |> PullRequestsSync.sync
    |> PullRequestsBroadcaster.broadcast(user)
  end

  def handle_cast({:fetch_user_pull_requests, fetch_params}, state) do
    %{user: user} = fetch_params
    {:noreply, fetch_and_update_pull_requests(user)}
  end

  def init([]) do
    {:ok, []}
  end

  def start_link(opts \\ []) do
    {:ok, pid} = GenServer.start_link(__MODULE__, [], opts)
  end
end

{'fraction_non_alphanumeric': 0.11930783242258652, 'fraction_numerical': 0.0024286581663630845, 'mean_word_length': 3.3816489361702127, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '244219', 'n_tokens_mistral': 1177, 'n_tokens_neox': 1124, 'n_words': 234}