<commit_before>require 'ostruct'

module NineGag
  class Scraper
    # path = ":section/:type"
    # next_page = "last_id"
    def self.index(path, next_page = nil)
      if next_page.nil?
        data = scrape_html(path)
      else
        data = generate_json_posts(path, next_page)
      end

      generate_index_data(data, next_page)
    end

    # path = "gag/:id"
    def self.show(path)
      generate_show_data(scrape_html(path).search('article').first)
    end

    private

    def self.full_url(path)
      "http://9gag.com/#{path}"
    end

    # will return Array of Nokogiri
    def self.generate_json_posts(path, next_page)
      items = JSON.parse(scrape_json(path, next_page).body)["items"]

      items.map { |item| scrape_html(item.last).search('article').first }
    end

    def self.scrape_json(path, next_page)
      RestClient.get(full_url(path),
        { Accept: 'application/json', "X-Requested-With": 'XMLHttpRequest',
          params: { id: next_page, c: 10 }
        }
      )
    end

    # scrape html or from path
    def self.scrape_html(path)
      Nokogiri::HTML(path_or_html(path))
    end

    # check parameter if path or html
    def self.path_or_html(data)
      # if path
      if data.length < 50
        URI.parse(data).read
      else
        data
      end
    end

    def self.generate_show_data(post, index = false)
      if index
        title = post.search('h2.badge-item-title .badge-evt')
      else
        title = post.search('h2.badge-item-title')
      end

      post_meta = post.search('p.post-meta').first

      post_data = {
        id: post.attribute('data-entry-id').value,
        title: title.text.strip,
        url: post.attribute('data-entry-url').value,
        image: image_data(post.search('div.badge-post-container a img').first),
        comments_count: post_meta.search('a.comment').first.text.sub(' comments', '').sub(',', '').strip.to_i,
        points: post_meta.search('a.point').first.text.sub(' points', '').sub(',', '').strip,
        media: media_data(post.search('video').first)
      }<commit_msg>fix scrape<commit_after>require 'ostruct'

module NineGag
  class Scraper
    # path = ":section/:type"
    # next_page = "last_id"
    def self.index(path, next_page = nil)
      if next_page.nil?
        data = scrape_html(path)
      else
        data = generate_json_posts(path, next_page)
      end

      generate_index_data(data, next_page)
    end

    # path = "gag/:id"
    def self.show(path)
      generate_show_data(scrape_html(path).search('article').first)
    end

    private

    def self.full_url(path)
      "http://9gag.com/#{path}"
    end

    # will return Array of Nokogiri
    def self.generate_json_posts(path, next_page)
      items = JSON.parse(scrape_json(path, next_page).body)["items"]

      items.map { |item| scrape_html(item.last).search('article').first }
    end

    def self.scrape_json(path, next_page)
      RestClient.get(full_url(path),
        { Accept: 'application/json', "X-Requested-With": 'XMLHttpRequest',
          params: { id: next_page, c: 10 }
        }
      )
    end

    # scrape html or from path
    def self.scrape_html(path)
      Nokogiri::HTML(path_or_html(path))
    end

    # check parameter if path or html
    def self.path_or_html(data)
      # if path
      if data.length < 50
        open full_url(data)
      else
        data
      end
    end

    def self.generate_show_data(post, index = false)
      if index
        title = post.search('h2.badge-item-title .badge-evt')
      else
        title = post.search('h2.badge-item-title')
      end

      post_meta = post.search('p.post-meta').first

      post_data = {
        id: post.attribute('data-entry-id').value,
        title: title.text.strip,
        url: post.attribute('data-entry-url').value,
        image: image_data(post.search('div.badge-post-container a img').first),
        comments_count: post_meta.search('a.comment').first.text.sub(' comments', '').sub(',', '').strip.to_i,
        points: post_meta.search('a.point').first.text.sub(' points', '').sub(',', '').strip,
        media: media_data(post.search('video').first)
      }
{'fraction_non_alphanumeric': 0.1378482228626321, 'fraction_numerical': 0.0033621517771373678, 'mean_word_length': 2.5029436501261566, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '776809', 'n_tokens_mistral': 1437, 'n_tokens_neox': 1392, 'n_words': 264}