<commit_before>#!/usr/bin/env python
# Copyright (c) 2010, <NAME>
# Released under the Don't Be A Douchbag License.
# Use responsibly. Contribute changes if you feel like it. No CP!

import urllib2
import os
import posixpath
import json
import time
from optparse import OptionParser
from BeautifulSoup import BeautifulSoup

options = OptionParser()
options.add_option("-b", "--board", dest="board", default='b')
options.add_option("-o", "--overwrite-images", dest="overwriteImages",
    default=True, help="Overwrite non-empty images", action="store_true")


class Post(object):
    def __init__(self, postID, text, poster, timestamp, image, imageTitle):
        self.id = postID
        self.text = text
        self.poster = poster
        self.timestamp = timestamp
        if image:
            self.image = posixpath.basename(image)
            self.imageURL = image
            self.imageTitle = imageTitle
        else:
            self.imageURL = self.image = self.imageTitle = None
    def __repr__(self):
        if self.image:
            return u"%(id)s by %(poster)s with %(image)s" % self.__dict__
        else:
            return u"%(id)s by %(poster)s with no image" % self.__dict__


def getSoup(board, thread):
    url = "http://boards.4chan.org/%s/res/%s" % (board, thread)
    print "downloading thread %s for board %s at %s" % (thread, board, url)
    f = urllib2.urlopen(url)
    soup = BeautifulSoup(f)
    f.close()
    return soup


def getPosts(soup):
    posts = []
    imageCount = 0
    postTables = (td.findParent('table')
        for td in soup.findAll('td', 'doubledash'))

    for postTable in postTables:
        postID = postTable.find('td', id=True)['id']
        text = postTable.find('blockquote').renderContents().strip()
        posterSpan = postTable.find('span', 'commentpostername')
        poster = posterSpan.string
        timestamp = posterSpan.findNext(text=True)

        filespan = postTable.find('span', 'filesize')
        if filespan:
            imageCount += 1
            imageURL = filespan.find('a')['href']
            imageTitle = postTable.find('span', title=True).string
        else:
            imageURL = None
            imageTitle = None

        post = Post(postID, text, poster, timestamp, imageURL, imageTitle)
        print u"found %s" % post
        posts.append(post)
    print u"found %i posts with %i images" % (len(posts), imageCount)
    return posts


def downloadImages(posts, dest, overwriteImages):
    imageDir = os.path.join(dest, "images")
    if not os.path.exists(imageDir):
        os.mkdir(imageDir)
    for post in posts:
        if post.image:
            localPath = os.path.join(imageDir, post.image)
            if os.path.exists(localPath):
                if overwriteImages and os.path.getsize(localPath) != 0:
                    print u"Skip: image %s already exists" % post.image
                    continue
            print u"downloading %s to %s" % (post.imageURL, post.image)
            with open(localPath, 'w') as f:
                remote = urllib2.urlopen(post.imageURL)
                f.write(remote.read())
            time.sleep(1) # be nice to the servers


def writeData(thread, posts, dest):
    target = os.path.join(dest, "thread.js")
    jsonPosts = []
    jsonCode = {}
    jsonCode['id'] = thread
    jsonCode['posts'] = jsonPosts
    for post in posts:
        jsonPosts.append({
            'id': post.id,
            'poster': post.poster,
            'image': post.image,
            'timestamp': post.timestamp,
            'text': post.text,
            'imageTitle': post.imageTitle,
            })
    print u"writing thread data for %s to %s" % (thread, target)
    with open(target, 'w') as f:
        json.dump(jsonCode, f)


def main():
    opts, args = options.parse_args()
    if len(args) != 2:
        print options.usage
    thread = args[0]
    baseDest = args[1]
    board = opts.board
    overwriteImages = opts.overwriteImages
    dest = os.path.join(baseDest, u"%s-%s" % (board, thread))
    if not os.path.exists(dest):
        os.makedirs(dest)
    soup = getSoup(opts.board, thread)
    posts = getPosts(soup)
    downloadImages(posts, dest, overwriteImages)
    writeData(thread, posts, dest)

if __name__ == "__main__":
    main()<commit_msg>Add thread updating until 404/keyboard interrupt<commit_after>#!/usr/bin/env python
# Copyright (c) 2010, <NAME>
# Released under the Don't Be A Douchbag License.
# Use responsibly. Contribute changes if you feel like it. No CP!

import urllib2
import os
import posixpath
import json
import time
from optparse import OptionParser
from BeautifulSoup import BeautifulSoup

options = OptionParser()
options.add_option("-b", "--board", dest="board", default='b')
options.add_option("-o", "--overwrite-images", dest="overwriteImages",
    default=False, help="Overwrite non-empty images", action="store_true")
options.add_option("-u", "--update", dest="update", action="store_true",
    default=False, help="update the thread")


class Post(object):
    def __init__(self, postID, text, poster, timestamp, image, imageTitle):
        self.id = postID
        self.text = text
        self.poster = poster
        self.timestamp = timestamp
        if image:
            self.image = posixpath.basename(image)
            self.imageURL = image
            self.imageTitle = imageTitle
        else:
            self.imageURL = self.image = self.imageTitle = None
    def __repr__(self):
        if self.image:
            return u"%(id)s by %(poster)s with %(image)s" % self.__dict__
        else:
            return u"%(id)s by %(poster)s with no image" % self.__dict__


def getSoup(board, thread):
    url = "http://boards.4chan.org/%s/res/%s" % (board, thread)
    print "downloading thread %s for board %s at %s" % (thread, board, url)
    f = urllib2.urlopen(url)
    soup = BeautifulSoup(f)
    f.close()
    return soup


def getPosts(soup):
    posts = []
    imageCount = 0
    postTables = (td.findParent('table')
        for td in soup.findAll('td', 'doubledash'))

    for postTable in postTables:
        postID = postTable.find('td', id=True)['id']
        text = postTable.find('blockquote').renderContents().strip()
        posterSpan = postTable.find('span', 'commentpostername')
        poster = posterSpan.string
        timestamp = posterSpan.findNext(text=True)

        filespan = postTable.find('span', 'filesize')
        if filespan:
            imageCount += 1
            imageURL = filespan.find('a')['href']
            imageTitle = postTable.find('span', title=True).string
        else:
            imageURL = None
            imageTitle = None

        post = Post(postID, text, poster, timestamp, imageURL, imageTitle)
        print u"found %s" % post
        posts.append(post)
    print u"found %i posts with %i images" % (len(posts), imageCount)
    return posts


def downloadImages(posts, dest, overwriteImages):
    imageDir = os.path.join(dest, "images")
    if not os.path.exists(imageDir):
        os.mkdir(imageDir)
    for post in posts:
        if post.image:
            localPath = os.path.join(imageDir, post.image)
            if os.path.exists(localPath):
                if not overwriteImages and os.path.getsize(localPath) != 0:
                    print u"Skip: image %s already exists" % post.image
                    continue
            print u"downloading %s to %s" % (post.imageURL, post.image)
            with open(localPath, 'w') as f:
                remote = urllib2.urlopen(post.imageURL)
                f.write(remote.read())
            time.sleep(1) # be nice to the servers


def writeData(thread, posts, dest):
    target = os.path.join(dest, "thread.js")
    jsonPosts = []
    jsonCode = {}
    jsonCode['id'] = thread
    jsonCode['posts'] = jsonPosts
    for post in posts:
        jsonPosts.append({
            'id': post.id,
            'poster': post.poster,
            'image': post.image,
            'timestamp': post.timestamp,
            'text': post.text,
            'imageTitle': post.imageTitle,
            })
    print u"writing thread data for %s to %s" % (thread, target)
    with open(target, 'w') as f:
        json.dump(jsonCode, f)


def main():
    opts, args = options.parse_args()
    if len(args) != 2:
        print options.usage
    thread = args[0]
    baseDest = args[1]
    board = opts.board
    overwriteImages = opts.overwriteImages
    if opts.update:
        updates = -1
    else:
        updates = 1
    dest = os.path.join(baseDest, u"%s-%s" % (board, thread))
    if not os.path.exists(dest):
        os.makedirs(dest)
    try:
        while updates != 0:
            updates -= 1
            soup = getSoup(opts.board, thread)
            posts = getPosts(soup)
            downloadImages(posts, dest, overwriteImages)
            writeData(thread, posts, dest)
            if updates != 0:
                print "-" * 40
                time.sleep(10)
    except KeyboardInterrupt:
        print "Keyboard Interrupt, ending archiving"
    except urllib2.HTTPError, e:
        if e.code == 404:
            print "Thread or image 404ed"
        else:
            raise

if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.12269272529858849, 'fraction_numerical': 0.005320304017372421, 'mean_word_length': 2.330079537237889, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '56006', 'n_tokens_mistral': 2849, 'n_tokens_neox': 2786, 'n_words': 746}