<commit_before>require 'funky/connections/base'
require 'json'

module Funky
  # @api private
  module Connection
    class API < Base
      def self.fetch(path_query, is_array: false)
        uri = URI "https://#{host}/v2.8/#{path_query}&limit=100&access_token=#{app_id}%7C#{app_secret}"
        is_array ? fetch_multiple_pages(uri) : json_for(uri)
      end

      def self.fetch_multiple_pages(uri)
        json = json_for(uri)
        if json[:paging][:next]
          next_paging_uri = URI json[:paging][:next]
          json[:data] + fetch_multiple_pages(next_paging_uri)
        else
          json[:data]
        end
      end

      def self.request(id:, fields:)
        uri = URI::HTTPS.build host: host,
          path: "/v2.8/#{id}",
          query: "access_token=#{app_id}%7C#{app_secret}&fields=#{fields}"
        response_for(get_http_request(uri), uri)
      end

      def self.batch_request(ids:, fields:)
        uri = URI::HTTPS.build host: host,
          path: "/",
          query: "include_headers=false&access_token=#{app_id}%7C#{app_secret}"
        batch = create_batch_for ids, fields
        http_request = post_http_request uri
        http_request.set_form_data batch: batch.to_json
        response_for(http_request, uri)
      end

    private

      def self.host
        'graph.facebook.com'
      end

      def self.app_id
        Funky.configuration.app_id
      end

      def self.app_secret
        Funky.configuration.app_secret
      end

      def self.post_http_request(uri)
        Net::HTTP::Post.new uri
      end

      def self.create_batch_for(ids, fields)
        ids.map do |id|
          {"method":"GET", "relative_url": "/v2.8/#{id}?fields=#{fields}"}
        end
      end
    end
  end
end
<commit_msg>Use timestamp for next paging of #videos<commit_after>require 'funky/connections/base'
require 'json'

module Funky
  # @api private
  module Connection
    class API < Base
      def self.fetch(path_query, is_array: false)
        uri = URI "https://#{host}/v2.8/#{path_query}&limit=100&access_token=#{app_id}%7C#{app_secret}"
        is_array ? fetch_multiple_pages(uri) : json_for(uri)
      end

      def self.fetch_multiple_pages(uri)
        json = json_for(uri)
        if json[:data].empty?
          []
        else
          timestamp = json[:data][-1][:created_time]
          if @previous_timestamp == timestamp
            []
          else
            @previous_timestamp = timestamp
            uri.query = uri.query + "&until=#{Time.parse(timestamp).to_i}"
            json[:data] + fetch_multiple_pages(uri)
          end
        end
      end

      def self.request(id:, fields:)
        uri = URI::HTTPS.build host: host,
          path: "/v2.8/#{id}",
          query: "access_token=#{app_id}%7C#{app_secret}&fields=#{fields}"
        response_for(get_http_request(uri), uri)
      end

      def self.batch_request(ids:, fields:)
        uri = URI::HTTPS.build host: host,
          path: "/",
          query: "include_headers=false&access_token=#{app_id}%7C#{app_secret}"
        batch = create_batch_for ids, fields
        http_request = post_http_request uri
        http_request.set_form_data batch: batch.to_json
        response_for(http_request, uri)
      end

    private

      def self.host
        'graph.facebook.com'
      end

      def self.app_id
        Funky.configuration.app_id
      end

      def self.app_secret
        Funky.configuration.app_secret
      end

      def self.post_http_request(uri)
        Net::HTTP::Post.new uri
      end

      def self.create_batch_for(ids, fields)
        ids.map do |id|
          {"method":"GET", "relative_url": "/v2.8/#{id}?fields=#{fields}"}
        end
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.11827669253411828, 'fraction_numerical': 0.00668985817500669, 'mean_word_length': 2.270341207349081, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4795695', 'n_tokens_mistral': 1230, 'n_tokens_neox': 1198, 'n_words': 230}