<commit_before>require "json"
require 'httparty'
require "iconv"

module CrowdFundingParser
  module Parser
    class Kickstarter < General
      include HTTParty
      def initialize
        # art = 1, comic = 3, game = 12,
        @category_ids = [1, 3, 6, 7, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 26]
        @parse_method = :doc
      end

      def platform_url
        "https://www.kickstarter.com"
      end

      def get_all_categories(status = "online")
        status_code = get_status_code(status)
        jsons = get_category_project_jsons(status_code)
        jsons.flatten.compact!
        categories = []
        Parallel.each(jsons, in_precesses: 2, in_threads: 5) do |json|
          category = { id: json["category"]["id"], name: json["category"]["name"], parent_id: json["category"]["parent_id"]}
          categories << category
        end
        categories.uniq
      end

      def get_project_links(status = "online")
        status_code = get_status_code(status)

        jsons = @category_ids.map do |category_id|
          category_jsons = get_category_project_jsons(status_code, category_id)
        end.flatten.compact

        Parallel.map(jsons, in_precesses: 2, in_threads: 5) do |json|
          unless json["state"] != "live" && json["pledged"].to_i == 0
            if json["state"] == status_code
              project_url = json["urls"]["web"]["project"]
            end
          end
        end
      end

      def get_category_project_jsons(status_code = "live", category_id = 0)
        jsons = []

        Parallel.each(1..200, in_precesses: 2, in_threads: 5) do |i|
          begin
            api_url = get_projects_page_api(i, status_code, category_id)
            json = get_json_through_url(api_url)["projects"]
            jsons << json
          rescue Exception => e
            puts e
            Parallel::Stop
          end
        end
        jsons
      end

      private

      def get_project_page_api(project_url)
        project_url.split("?").first + ".json"
      end

      def get_projects_page_api(page = 1, status_code = "live", category_id = 0)
        "https://www.kickstarter.com/projects/search.json?page=#{page}&state=#{status_code}&category_id=#{category_id}"
      end

      def get_project_search_result_api(name)
        "https://www.kickstarter.com/projects/search.json?term=#{name}"
      end

      def get_status_code(status)
        case status<commit_msg>canceled output when exception raised<commit_after>require "json"
require 'httparty'
require "iconv"

module CrowdFundingParser
  module Parser
    class Kickstarter < General
      include HTTParty
      def initialize
        # art = 1, comic = 3, game = 12,
        @category_ids = [1, 3, 6, 7, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 26]
        @parse_method = :doc
      end

      def platform_url
        "https://www.kickstarter.com"
      end

      def get_all_categories(status = "online")
        status_code = get_status_code(status)
        jsons = get_category_project_jsons(status_code)
        jsons.flatten.compact!
        categories = []
        Parallel.each(jsons, in_precesses: 2, in_threads: 5) do |json|
          category = { id: json["category"]["id"], name: json["category"]["name"], parent_id: json["category"]["parent_id"]}
          categories << category
        end
        categories.uniq
      end

      def get_project_links(status = "online")
        status_code = get_status_code(status)

        jsons = @category_ids.map do |category_id|
          category_jsons = get_category_project_jsons(status_code, category_id)
        end.flatten.compact

        Parallel.map(jsons, in_precesses: 2, in_threads: 5) do |json|
          unless json["state"] != "live" && json["pledged"].to_i == 0
            if json["state"] == status_code
              project_url = json["urls"]["web"]["project"]
            end
          end
        end
      end

      def get_category_project_jsons(status_code = "live", category_id = 0)
        jsons = []

        Parallel.each(1..200, in_precesses: 2, in_threads: 5) do |i|
          begin
            api_url = get_projects_page_api(i, status_code, category_id)
            json = get_json_through_url(api_url)["projects"]
            jsons << json
          rescue Exception => e
            Parallel::Stop
          end
        end
        jsons
      end

      private

      def get_project_page_api(project_url)
        project_url.split("?").first + ".json"
      end

      def get_projects_page_api(page = 1, status_code = "live", category_id = 0)
        "https://www.kickstarter.com/projects/search.json?page=#{page}&state=#{status_code}&category_id=#{category_id}"
      end

      def get_project_search_result_api(name)
        "https://www.kickstarter.com/projects/search.json?term=#{name}"
      end

      def get_status_code(status)
        case status
{'fraction_non_alphanumeric': 0.11254348270922857, 'fraction_numerical': 0.017597708205443013, 'mean_word_length': 2.3502398903358466, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 5, 'https://': 6, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2393660', 'n_tokens_mistral': 1622, 'n_tokens_neox': 1519, 'n_words': 327}