<commit_before>require 'pry'

module Mumblr
  class Blog < Model
    include DataMapper::Resource

    property :id, Serial
    property :url, String
    property :name, String
    property :posts_retrieved_at, DateTime
    property :created_at, DateTime

    has n, :posts
    has n, :likes

    def self.retrieve(base_hostname, tumblr_params)
      base_hostname = normalize_base_hostname(base_hostname)
      Model::logger.debug("Resolved blog to base hostname: #{base_hostname}")
      blog = Blog.first_or_create({name: base_hostname}, api_hash(base_hostname))
      Post.retrieve(blog, tumblr_params)
      Model::logger.debug("Got blog: #{blog}")
      blog
    end

    def self.api_hash(name)
      Model::logger.debug("Retrieving from API for hostname: #{name}")
      api_hash = Model.client.blog_info(name)
      {
        url: api_hash['blog']['url'],
        name: api_hash['blog']['name'],
        created_at:  DateTime.now
      }
    end

    # options:
    #
    # type: one of: text, quote, link, answer, video, audio, photo, chat
    # after_time:
    # before_time:
    # count:
    def likes_posts_contents(options={})
      tumblr_params = {
        offset: 0,
        limit: 20
      }
      unless @raw_likes
        @raw_likes = []
        loop do
          STDERR.puts "So far we have #{@raw_likes.length}"
          tumblr_params[:offset] = @raw_likes.length
          likes_res = client.blog_likes(@base_hostname, tumblr_params)<commit_msg>Update how blog gets posts<commit_after>require 'pry'

module Mumblr
  class Blog < Model
    include DataMapper::Resource

    property :id, Serial
    property :url, String
    property :name, String
    property :posts_retrieved_at, DateTime
    property :created_at, DateTime

    has n, :posts
    has n, :likes

    def self.retrieve(base_hostname, tumblr_params)
      base_hostname = normalize_base_hostname(base_hostname)
      Model::logger.debug("Resolved blog to base hostname: #{base_hostname}")
      blog = Blog.first_or_create({name: base_hostname}, api_hash(base_hostname))
      Post.retrieve_from_blog(blog, tumblr_params)
      Model::logger.debug("Got blog: #{blog}")
      blog
    end

    def self.api_hash(name)
      Model::logger.debug("Retrieving from API for hostname: #{name}")
      api_hash = Model.client.blog_info(name)
      {
        url: api_hash['blog']['url'],
        name: api_hash['blog']['name'],
        created_at:  DateTime.now
      }
    end

    def self.retrieve_likes(base_hostname, options={})
      base_hostname = normalize_base_hostname(base_hostname)
      Model::logger.debug("")
      blog = Blog.first_or_create({name: base_hostname}, api_hash(base_hostname))
      Post.retrieve_from_blog(blog, tumblr_params)
      Model::logger.debug("Got blog: #{blog}")
      blog
    end

    # options:
    #
    # type: one of: text, quote, link, answer, video, audio, photo, chat
    # after_time:
    # before_time:
    # count:
    def likes_posts_contents(options={})
      tumblr_params = {
        offset: 0,
        limit: 20
      }
      unless @raw_likes
        @raw_likes = []
        loop do
          STDERR.puts "So far we have #{@raw_likes.length}"
          tumblr_params[:offset] = @raw_likes.length
          likes_res = client.blog_likes(@base_hostname, tumblr_params)
{'fraction_non_alphanumeric': 0.11127910492893861, 'fraction_numerical': 0.0018143332325370426, 'mean_word_length': 2.891764705882353, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7358989', 'n_tokens_mistral': 1070, 'n_tokens_neox': 1059, 'n_words': 234}