<commit_before>defmodule WhistlerNewsReader.Article do

  alias WhistlerNewsReader.FeedFetcher

  def scrape(url) do
    case FeedFetcher.fetch(url) do
      {:ok, body} ->
        scrape_article(body)
      other ->
        other
    end
  end

  def scrape_article(body) do
    body
    |> Floki.find("article, p, body")
    |> Enum.map(fn x -> Floki.text(x, deep: false) end)
    |> Enum.map(&normalize_whitespace/1)
    |> Enum.filter(&is_relevant?/1)
    |> Enum.join("\n\n")
  end

  def normalize_whitespace(text) do
    text
    |> String.replace(~r/\s+/, " ")
    |> String.replace(~r/\s+/, " ")
    |> String.strip
  end

  def is_relevant?(text) do
    (String.length(text) > 30) &&
    (String.contains?(text, [". ","? ", "! ", "\" ", "\", ", ": "]))
  end

  defp attr(element, name) do
    Enum.at(Floki.attribute(element, name), 0)
  end
end<commit_msg>scrape article feature<commit_after>defmodule WhistlerNewsReader.Article do

  alias WhistlerNewsReader.FeedFetcher

  def scrape(url) do
    case FeedFetcher.fetch(url) do
      {:ok, body} ->
        scrape_article(body)
      other ->
        other
    end
  end

  def scrape_article(body) do
    body
    |> Floki.find("article, .article, .content, .entry, .main, .post, .text")
    |> Floki.find("figure, p, q, blockquote, h1, h2")
    |> Enum.map(fn x ->
      case List.first(Tuple.to_list(x)) do
        "figure" ->
          Floki.raw_html(x)
        "p" ->
          if is_relevant?(Floki.text(x, deep: false)) do
            Floki.raw_html(x)
          else
            ""
          end
        other ->
          ""
      end
    end)
    |> Enum.join
  end

  def is_relevant?(text) do
    (String.length(text) > 30) &&
    (String.contains?(text, [". ","? ", "! ", ", ", "; ", ": "]))
  end

end
{'fraction_non_alphanumeric': 0.16591676040494938, 'fraction_numerical': 0.005061867266591676, 'mean_word_length': 2.240437158469945, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4763382', 'n_tokens_mistral': 668, 'n_tokens_neox': 635, 'n_words': 124}