<commit_before>require 'monkeyshines/scrape_request/raw_json_contents'
module Wuclan
  module Twitter
    module Scrape
      # Effectively unlimited request maximum
      # NO_LIMIT = 2**31
      NO_LIMIT = (50_000_000 / 100) # controlled insanity

      #
      # Base class for twitter API requests
      #
      class Base < TypedStruct.new(
          [:twitter_user_id,  Integer],
          [:page,             Integer],
          [:moreinfo,         String],
          [:url,              String],
          [:scraped_at,       Bignum],
          [:response_code,    Integer],
          [:response_message, String],
          [:contents,         String]
          )
        class_inheritable_accessor :resource_path, :page_limit, :max_items

        # Let us be peers with AFollowsB and TwitterUser and etc.
        include Wuclan::Twitter::Model
        # Contents are JSON
        include Monkeyshines::RawJsonContents
        # Requests are paginated
        include Monkeyshines::ScrapeJob::Paginated

        #<commit_msg>Eliminated spurious reference to Monkeyshines::ScrapeJob that won't resolve at the moment, as it's not currently used in this source file but potentially will be reintroduced soonish (so suggested flip).<commit_after>require 'monkeyshines/scrape_request/raw_json_contents'
module Wuclan
  module Twitter
    module Scrape
      # Effectively unlimited request maximum
      # NO_LIMIT = 2**31
      NO_LIMIT = (50_000_000 / 100) # controlled insanity

      #
      # Base class for twitter API requests
      #
      class Base < TypedStruct.new(
          [:twitter_user_id,  Integer],
          [:page,             Integer],
          [:moreinfo,         String],
          [:url,              String],
          [:scraped_at,       Bignum],
          [:response_code,    Integer],
          [:response_message, String],
          [:contents,         String]
          )
        class_inheritable_accessor :resource_path, :page_limit, :max_items

        # Let us be peers with AFollowsB and TwitterUser and etc.
        include Wuclan::Twitter::Model
        # Contents are JSON
        include Monkeyshines::RawJsonContents
        # Requests are paginated
        #DMPinclude Monkeyshines::ScrapeJob::Paginated

        #
{'fraction_non_alphanumeric': 0.07743362831858407, 'fraction_numerical': 0.012389380530973451, 'mean_word_length': 2.140277777777778, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2470506', 'n_tokens_mistral': 666, 'n_tokens_neox': 606, 'n_words': 168}