<commit_before>require "tumblr_to_dayone/version"
require 'tumblr_to_dayone/tumblr'
require 'tumblr_to_dayone/dayone'
require 'tempfile'

module TumblrToDayone

  #   options:
  #
  # password                    - Password for blog (if there is one)
  # filter                      - Only get Tumblr posts based on type (text, quote, photo, link, chat, video, or audio)
  # automatically_add_each_post - Automatically add each Tumblr post or ask before adding each one (defaults to false)
  # journal_path                - Location of Day One Journal file
  #

  def self.add_tumblr_posts_to_dayone(blog, options={})
    begin
      post_index = 0
      no_more_posts = false

      while !no_more_posts
        Tumblr.posts(blog, password = options[:password], :start => post_index, :type => options[:filter]) do |posts, total_posts|
          post_index += posts.count
          no_more_posts = posts.empty? || post_index >= total_posts

          exited = false

          posts.each do |post|
            unless exited
              post_status = options[:automatically_add_each_post] ? :y : yield(post)
              
              if post_status == :y || post_status == :s
                post_created = post.add_to_dayone!(starred = post_status == :s, dayone_journal = options[:journal_path])
<commit_msg>require open-uri to fix opening image urls<commit_after>require "tumblr_to_dayone/version"
require 'tumblr_to_dayone/tumblr'
require 'tumblr_to_dayone/dayone'
require 'tempfile'
require 'open-uri'

module TumblrToDayone

  #   options:
  #
  # password                    - Password for blog (if there is one)
  # filter                      - Only get Tumblr posts based on type (text, quote, photo, link, chat, video, or audio)
  # automatically_add_each_post - Automatically add each Tumblr post or ask before adding each one (defaults to false)
  # journal_path                - Location of Day One Journal file
  #

  def self.add_tumblr_posts_to_dayone(blog, options={})
    begin
      post_index = 0
      no_more_posts = false

      while !no_more_posts
        Tumblr.posts(blog, password = options[:password], :start => post_index, :type => options[:filter]) do |posts, total_posts|
          post_index += posts.count
          no_more_posts = posts.empty? || post_index >= total_posts

          exited = false

          posts.each do |post|
            unless exited
              post_status = options[:automatically_add_each_post] ? :y : yield(post)
              
              if post_status == :y || post_status == :s
                post_created = post.add_to_dayone!(starred = post_status == :s, dayone_journal = options[:journal_path])

{'fraction_non_alphanumeric': 0.0875748502994012, 'fraction_numerical': 0.0007485029940119761, 'mean_word_length': 2.568758344459279, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '479137', 'n_tokens_mistral': 831, 'n_tokens_neox': 772, 'n_words': 218}