<commit_before>require 'open-uri'

require_relative '../config/environment'

class HackerNewsCollector
  def perform(url)
	return if (id = url.match(/[0-9]+$/).to_s).blank?
	data_url = "https://hn.algolia.com/api/v1/items/#{id}"
	data = JSON.parse(open(data_url).read)

	discussion = Discussion.create(
	  title: data["title"],
	  url: url,
      source: "Hacker News"
	)
	save_the_children(discussion, nil, data["children"])

    DiscussionAnalyzer.new.perform(discussion)
  end

  handle_asynchronously :perform

  private

  def save_the_children(discussion, parent, children)
	children.each do |c|
	  next unless c["text"]
	  comment = Comment.create(
        discussion: discussion,
        parent: parent,
		text: Nokogiri::HTML(c["text"]).css('p').map(&:text).join(" "),
        user: c["author"],
        votes: c["points"]
      )
	  save_the_children(discussion, comment, c["children"])
	end
  end
end
<commit_msg>Preserve newlines in HN comments<commit_after>require 'open-uri'

require_relative '../config/environment'

class HackerNewsCollector
  def perform(url)
	return if (id = url.match(/[0-9]+$/).to_s).blank?
	data_url = "https://hn.algolia.com/api/v1/items/#{id}"
	data = JSON.parse(open(data_url).read)

	discussion = Discussion.create(
	  title: data["title"],
	  url: url,
      source: "Hacker News"
	)
	save_the_children(discussion, nil, data["children"])

    DiscussionAnalyzer.new.perform(discussion)
  end

  handle_asynchronously :perform

  private

  def save_the_children(discussion, parent, children)
	children.each do |c|
	  next unless c["text"]
	  comment = Comment.create(
        discussion: discussion,
        parent: parent,
		text: Nokogiri::HTML(c["text"]).css('p').map(&:text).join("\n"),
        user: c["author"],
        votes: c["points"]
      )
	  save_the_children(discussion, comment, c["children"])
	end
  end
end

{'fraction_non_alphanumeric': 0.15364025695931477, 'fraction_numerical': 0.0032119914346895075, 'mean_word_length': 3.8545454545454545, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '817220', 'n_tokens_mistral': 660, 'n_tokens_neox': 622, 'n_words': 126}