<commit_before>require 'net/http'
require 'uri'
require "json"

# Reddit content crawler
class Crawler

  REDDIT_ADDRESS = "https://www.reddit.com/"

  # @param keyword [String] a target keyword interpreted
  #   as a certain subreddit r/keyword
  # @example
  #   Crawler.new # fetch links in https://www.reddit.com/new.json
  #   Crawler.new("cute") # fetch links in https://www.reddit.com/r/cute
  def initialize(keyword='new')
    base_address = (keyword == 'new') ? REDDIT_ADDRESS : "#{REDDIT_ADDRESS}r/"
    url = "#{base_address}#{keyword}/.json"
    page_as_json = open_as_json(url)
    @links = extract_images(page_as_json) unless page_as_json.empty?
  end

  # does links contain meaningful fetched data
  def ok?
    !@links.nil?
  end

  # Fetched crawler links wither their tile
  #
  # @return [Hash{PostTitle => PostUrl}]
  def links
    Hash[*@links.flatten]
  end

  protected

  # Extract content of posts from target reddit/subreddit.
  #
  # @info: The title and the relevant link to the post is
  #   extracted.
  # @param json_hash [Hash] json hash of a target reddit page.
  # @return [Array<Array[String]] a list containing the extracted
  #   title and links of a particular reddit post.
  def extract_images(json_hash)
    json_hash['data']['children'].map do |node|
      post = node['data']
      title = post['title']
      [title, fetched_content_of(post)]
    end
  end

  # Extract youtube link or permalink from current post.
  def fetched_content_of(post)
    case post['domain']
    when 'youtube'
      post["secure_media"]['oembed']["url"]
    else
      post['permalink']
    end
  end

  # Get json representation of target url.
  #
  # @param json_url [String] url to json response of target reddit page.
  # @info: corresponds to the crawler's url concatinated with ".json"
  # @return [Hash] containing the requested json object.
  def open_as_json(json_url)
    uri = URI.parse(json_url)
    response = Net::HTTP.get_response(uri)
    return {} unless response.code == "200"
    JSON.parse(response.body)
  end

end

# Fetches image links from /r/pics
class RedditImgCrawler < Crawler
  def initialize
    super('pics')
  end

  protected

  def fetched_content_of(post)
    post['url']
  end

end
<commit_msg>Randomly shuffle a crawler's fetched content<commit_after>require 'net/http'
require 'uri'
require "json"

# Reddit content crawler
class Crawler

  REDDIT_ADDRESS = "https://www.reddit.com/"

  # @param keyword [String] a target keyword interpreted
  #   as a certain subreddit r/keyword
  # @example
  #   Crawler.new # fetch links in https://www.reddit.com/new.json
  #   Crawler.new("cute") # fetch links in https://www.reddit.com/r/cute
  def initialize(keyword='new')
    base_address = (keyword == 'new') ? REDDIT_ADDRESS : "#{REDDIT_ADDRESS}r/"
    url = "#{base_address}#{keyword}/.json"
    page_as_json = open_as_json(url)
    @links = extract_images(page_as_json) unless page_as_json.empty?
  end

  # Checks whether the crawler could fetch and store meanungfu data in links.
  #
  # @return [Boolean] true if fetching web-content yield status 200,
  #   otherwise false.
  def ok?
    !@links.nil?
  end

  # Retrieve a hash containing a crawler's fetched data.
  #
  # @info: links are randomly re-sorted.
  # @return [Hash{PostTitle => PostUrl}] fetched crawler links with their title
  def links
    randomly_sorted_links = @links.shuffle
    Hash[*randomly_sorted_links.flatten]
  end

  protected

  # Extract content of posts from target reddit/subreddit.
  #
  # @info: The title and the relevant link to the post is
  #   extracted.
  # @param json_hash [Hash] json hash of a target reddit page.
  # @return [Array<Array[String]] a list containing the extracted
  #   title and links of a particular reddit post.
  def extract_images(json_hash)
    json_hash['data']['children'].map do |node|
      post = node['data']
      title = post['title']
      [title, fetched_content_of(post)]
    end
  end

  # Extract youtube link or permalink from current post.
  def fetched_content_of(post)
    case post['domain']
    when 'youtube'
      post["secure_media"]['oembed']["url"]
    else
      post['permalink']
    end
  end

  # Get json representation of target url.
  #
  # @param json_url [String] url to json response of target reddit page.
  # @info: corresponds to the crawler's url concatinated with ".json"
  # @return [Hash] containing the requested json object.
  def open_as_json(json_url)
    uri = URI.parse(json_url)
    response = Net::HTTP.get_response(uri)
    return {} unless response.code == "200"
    JSON.parse(response.body)
  end

end

# Fetches image links from /r/pics
class RedditImgCrawler < Crawler
  def initialize
    super('pics')
  end

  protected

  def fetched_content_of(post)
    post['url']
  end

end

{'fraction_non_alphanumeric': 0.10674974039460021, 'fraction_numerical': 0.001869158878504673, 'mean_word_length': 3.434622467771639, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 6, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6836522', 'n_tokens_mistral': 1599, 'n_tokens_neox': 1583, 'n_words': 469}