<commit_before>module JekyllImport
  module Importers
    class Tumblr < Importer
      def self.require_deps
        JekyllImport.require_with_fallback(%w[
          rubygems
          fileutils
          open-uri
          nokogiri
          json
          uri
          time
          jekyll
        ])
      end

      def self.specify_options(c)
        c.option 'url', '--url URL', 'Tumblr URL'
        c.option 'format', '--format FORMAT', 'Output format (default: "html")'
        c.option 'grab_images', '--grab_images', 'Whether to grab images (default: false)'
        c.option 'add_highlights', '--add_highlights', 'Whether to add highlights (default: false)'
        c.option 'rewrite_urls', '--rewrite_urls', 'Whether to rewrite URLs (default: false)'
      end

      def self.process(options)
        url            = options.fetch('url')
        format         = options.fetch('format', "html")
        grab_images    = options.fetch('grab_images', false)
        add_highlights = options.fetch('add_highlights', false)
        rewrite_urls   = options.fetch('rewrite_urls', false)

        @grab_images = grab_images
        FileUtils.mkdir_p "_posts/tumblr"
        url += "/api/read/json/"
        per_page = 50
        posts = []
        # Two passes are required so that we can rewrite URLs.
        # First pass builds up an array of each post as a hash.
        begin
          current_page = (current_page || -1) + 1
          feed_url = url + "?num=#{per_page}&start=#{current_page * per_page}"
          puts "Fetching #{feed_url}"
          feed = open(feed_url)
          json = feed.readlines.join("\n")[22...-1]  # Strip Tumblr's JSONP chars.
          blog = JSON.parse(json)
          puts "Page: #{current_page + 1} - Posts: #{blog["posts"].size}"
          batch = blog["posts"].map { |post| post_to_hash(post, format) }

          # If we're rewriting, save the posts for later.  Otherwise, go ahead and
          # dump these to disk now
          if rewrite_urls<commit_msg>Tumblr: look up beginning and end of JSON dynamically<commit_after>module JekyllImport
  module Importers
    class Tumblr < Importer
      def self.require_deps
        JekyllImport.require_with_fallback(%w[
          rubygems
          fileutils
          open-uri
          nokogiri
          json
          uri
          time
          jekyll
        ])
      end

      def self.specify_options(c)
        c.option 'url', '--url URL', 'Tumblr URL'
        c.option 'format', '--format FORMAT', 'Output format (default: "html")'
        c.option 'grab_images', '--grab_images', 'Whether to grab images (default: false)'
        c.option 'add_highlights', '--add_highlights', 'Whether to add highlights (default: false)'
        c.option 'rewrite_urls', '--rewrite_urls', 'Whether to rewrite URLs (default: false)'
      end

      def self.process(options)
        url            = options.fetch('url')
        format         = options.fetch('format', "html")
        grab_images    = options.fetch('grab_images', false)
        add_highlights = options.fetch('add_highlights', false)
        rewrite_urls   = options.fetch('rewrite_urls', false)

        @grab_images = grab_images
        FileUtils.mkdir_p "_posts/tumblr"
        url += "/api/read/json/"
        per_page = 50
        posts = []
        # Two passes are required so that we can rewrite URLs.
        # First pass builds up an array of each post as a hash.
        begin
          current_page = (current_page || -1) + 1
          feed_url = url + "?num=#{per_page}&start=#{current_page * per_page}"
          puts "Fetching #{feed_url}"
          feed = open(feed_url)
          contents = feed.readlines.join("\n")
          beginning = contents.index("{")
          ending = contents.rindex("}")
          json = feed.readlines.join("\n")[beginning...ending]  # Strip Tumblr's JSONP chars.
          blog = JSON.parse(json)
          puts "Page: #{current_page + 1} - Posts: #{blog["posts"].size}"
          batch = blog["posts"].map { |post| post_to_hash(post, format) }

          # If we're rewriting, save the posts for later.  Otherwise, go ahead and
          # dump these to disk now
          if rewrite_urls
{'fraction_non_alphanumeric': 0.11560418648905804, 'fraction_numerical': 0.003092293054234063, 'mean_word_length': 2.0873715124816448, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1350864', 'n_tokens_mistral': 1284, 'n_tokens_neox': 1240, 'n_words': 337}