<commit_before>#!/usr/bin/python

from datetime import datetime, MINYEAR
import pytumblr
from secrets import consumer_key, secret_key, oauth_token, oauth_secret

import django
from django.conf import settings
from django.db import connection
import sys
import os

sys.path.insert(0, "../../website")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
django.setup()

from noveltorpedo import models

# These globals will be initialized later.
client = None   # Tumblr API client
host = None     # Tumblr Host object in the database

def get_info(blog):
    """
    Fetch general information about the blog being tracked. Returns a
    dictionary.  Keys which are likely to be useful:
        title
        url
        description
        is_nsfw (boolean)
        updated (most recent post's timestamp, in integer seconds since epoch)
    """
    return client.blog_info(blog)["blog"]

def get_posts(blog, offset=0, limit=20):
    """
    Fetch post contents and metadata. Parameters:
        offset (how many posts back to begin)
        limit (how many posts to return; 20 is the max allowed by the API)
    Returns a list of dictionaries. Keys which are likely to be useful:
        blog_name
        body
        post_url
        short_url
        timestamp
        title
    """
    return client.posts(blog, type="text", filter="text",
                        offset=offset, limit=limit)["posts"]

def get_host():
    """
    Create the Tumblr host entry in the database iff it doesn't exist.
    In either case, returns the appropriate Host object, as well as saving
    it in a global variable so we don't have to query for it again.
    """
    global host
    if not host:
        host = models.Host.objects.filter(url = "tumblr.com").first()
    if not host:
        host = models.Host()
        host.url = "tumblr.com"
        host.spider = "scrapers/tumblr/fetch_tumblr.py"
        host.wait = 1 # per their robots.txt
        host.save()
    return host

def make_story(blog):
    """
    Add a new story to the database. Takes a Tumblr blog name and returns
    the corresponding Story object. Does no verification that the story
    hasn't already been added, and returns nothing.
    """
    blog_info = client.blog_info(blog)["blog"]
    story = models.Story()
    story.title = blog_info["title"]
    story.save()

    author = models.Author()
    author.name = blog
    author.save()
    story.authors.add(author)

    storyhost = models.StoryHost()
    storyhost.host = get_host()
    storyhost.url = blog + ".tumblr.com"
    storyhost.story = story
    storyhost.last_scraped = datetime(MINYEAR, 1, 1)
    storyhost.save()

if __name__ == "__main__":
    """
    This allows you to pass the short name of a tumblr on the command line to
    add it as a new entry to the story database. Doesn't retrieve posts yet.
    """
    import sys
    client = pytumblr.TumblrRestClient(consumer_key, secret_key, oauth_token,
                                       oauth_secret)
    try:
        blog = sys.argv[1]
        info = get_info(blog)
        posts = get_posts(blog)
    except IndexError:
        print("Please supply a tumblr name.")
        sys.exit(1)
    except KeyError:
        print("Tumblr not found.")
        sys.exit(1)

    make_story(blog)
<commit_msg>Add post retrieval and insertion.<commit_after>#!/usr/bin/python

from datetime import datetime
from pytz import utc
from time import sleep

import pytumblr
from secrets import consumer_key, secret_key, oauth_token, oauth_secret

import django
from django.conf import settings
from django.db import connection
import sys
import os

sys.path.insert(0, "../../website")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
django.setup()

from noveltorpedo import models

# These globals will be initialized later.
client = None   # Tumblr API client
host = None     # Tumblr Host object in the database

# This is defined by Tumblr's API.
MAX_POSTS = 20

def get_host():
    """
    Create the Tumblr host entry in the database iff it doesn't exist.
    In either case, returns the appropriate Host object, as well as saving
    it in a global variable so we don't have to query for it again.
    """
    global host
    if not host:
        try:
            host = models.Host.objects.get(url = "tumblr.com")
        except models.Host.DoesNotExist:
            host = models.Host()
            host.url = "tumblr.com"
            host.spider = "scrapers/tumblr/fetch_tumblr.py"
            host.wait = 1 # per their robots.txt
            host.save()
    return host

def create_story(blog):
    """
    Add a new story to the database. Takes a Tumblr blog name and returns
    the corresponding Story object. Does no verification that the story
    hasn't already been added, and returns nothing.
    """
    blog_info = client.blog_info(blog)["blog"]
    story = models.Story()
    story.title = blog_info["title"]
    story.save()

    author = models.Author()
    author.name = blog
    author.save()
    story.authors.add(author)

    storyhost = models.StoryHost()
    storyhost.host = get_host()
    storyhost.url = blog + ".tumblr.com"
    storyhost.story = story
    storyhost.last_scraped = datetime.min.replace(tzinfo=utc)
    storyhost.save()

    print story
    print "---"

def update_story(blog):
    """
    Retrieve any posts which have been added to a story since its last
    scraped timestamp. Assumes that the story is already in the database,
    and there are text posts on the blog. Returns nothing.
    """
    storyhost = models.StoryHost.objects.get(url = blog + ".tumblr.com")
    oldest_new = datetime.now(utc)
    offset = 0
    posts = get_posts(blog)
    post = posts.pop(0)
    post_date = datetime.fromtimestamp(post["timestamp"], utc)
    while post_date > storyhost.last_scraped:
        segment = models.StorySegment()
        segment.story = storyhost.story
        segment.title = post["title"]
        segment.contents = post["body"]
        segment.published = oldest_new
        segment.save()
        print segment
        try:
            post = posts.pop(0)
        except IndexError:
            sleep(get_host().wait)
            offset = offset + MAX_POSTS
            posts = get_posts(blog, offset)
            if not posts:
                break
            post = posts.pop(0)
        post_date = datetime.fromtimestamp(post["timestamp"], utc)
    storyhost.last_scraped = datetime.now(utc)
    storyhost.save()

def get_posts(blog, offset=0, limit=MAX_POSTS):
    """
    Fetch post contents and metadata. Parameters:
        offset (how many posts back to begin)
        limit (how many posts to return; 20 is the max allowed by the API)
    Returns a list of dictionaries.
    TODO: Move the wait timer over here.
    """
    return client.posts(blog, type="text", filter="text",
                        offset=offset, limit=limit)["posts"]

if __name__ == "__main__":
    """
    This allows you to pass the short name of a tumblr on the command line to
    add it to the database, including its text posts.
    """
    import sys
    client = pytumblr.TumblrRestClient(consumer_key, secret_key, oauth_token,
                                       oauth_secret)
    try:
        blog = sys.argv[1]
        posts = get_posts(blog)
    except IndexError:
        print("Please supply a tumblr name.")
        sys.exit(1)
    except KeyError:
        print("Tumblr not found.")
        sys.exit(1)

    create_story(blog)
    update_story(blog)

{'fraction_non_alphanumeric': 0.0901955567380604, 'fraction_numerical': 0.0034588266595716377, 'mean_word_length': 2.5279211637728767, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2246176', 'n_tokens_mistral': 2269, 'n_tokens_neox': 2163, 'n_words': 734}