<commit_before>#!/usr/bin/env python
#
# The MIT License
# 
# Copyright (c) 2010 <NAME>
# 
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# 
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# 
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

# requires json module

import os
import sys
import json
import getopt
import urllib2, urllib
from multiprocessing import Pool

def main():
    try:
        opts, args = getopt.getopt(sys.argv[1:], "t:h", ["token", "help"])
    except getopt.GetoptError, err:
        # print help information and exit:
        print str(err) # will print something like "option -a not recognized"
        usage()
        sys.exit(2)
        
    if not len(opts):
        usage()
        sys.exit(2)
    
    for o, a in opts:
        if o in ("-t", "--token"):
            token = a
            scrape_photos(token)
        else:
            usage()

def usage():
    print """
    python scrapebook.py -t AUTH_TOKEN
    
    To get your authtoken, head over to http://developers.facebook.com/docs/api,
    click on the https://graph.facebook.com/me/photos link, and copy the auth
    token in the url to the command line
    """
    
def save_photo(url, filename):
    print "Saving" + filename
    try:
        urllib.urlretrieve(url, filename)
    except urllib.HTTPError:
        print "Could not open url:%s" % url 
    
def scrape_photos(token):
    # Setup directory to store photos
    scrape_dir = os.path.join(os.curdir,"facebook")
    
    try:
        os.mkdir(scrape_dir)
    except OSError:
        # Folder already exists, continue on
        pass
    
    limit = 10000 #Too big? Nah.....
    url = "http://graph.facebook.com/me/photos?access_token=%s&limit=%d" % \
        (token, limit)
    try:
        data = urllib2.urlopen(url)
    except urllib2.HTTPError:
        print "Please check to make sure you have properly copied over your auth token and put it in quotes"
        sys.exit(1);
        
    graph = json.loads(data.read())
    
    pool = Pool(processes=25)
    
    for i, photo in enumerate(graph["data"]):
        purl = photo["source"]
        filename = os.path.join(scrape_dir, "facebook_%s.jpg" % i)
        pool.apply_async(save_photo, [purl, filename])
        
    pool.close()
    pool.join()
        

<commit_msg>Added support for downloading all albums. Need to remove the django dependency<commit_after>#!/usr/bin/env python
#
# The MIT License
# 
# Copyright (c) 2010 <NAME>
# 
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# 
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# 
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

# requires json module

import os
import sys
import json
import getopt
import urllib2, urllib
from multiprocessing import Pool

# Need to remove this dependency 
from django.template.defaultfilters import slugify


def usage():
    print """
    python scrapebook.py -t AUTH_TOKEN
    
    To get your authtoken, head over to http://developers.facebook.com/docs/api,
    click on the https://graph.facebook.com/me/photos link, and copy the auth
    token in the url to the command line
    """
    
def save_photo(url, filename, debug):
    if debug:
        print "Saving" + filename
    try:
        urllib.urlretrieve(url, filename)
    except urllib.HTTPError:
        print "Could not open url:%s" % url

class Scrapebook(object):
    
    def __init__(self):
        self.token = None
        self.debug = False
        self.base = "https://graph.facebook.com"
        
    def api_request(self, path, limit=10000):
        limit = 10000 #Too big? Nah.....
        url = "https://graph.facebook.com/%s?access_token=%s&limit=%d" % \
            (path, self.token, limit)
        try:
            data = urllib2.urlopen(url)
            data = json.loads(data.read())
        except urllib2.HTTPError:
            data = {}
            
        return data
            
        
        
    
    def setup(self):
        scrape_dir = os.path.join(os.curdir,"facebook")
    
        try:
            print "Creating facebook directory...."
            os.mkdir(scrape_dir)
        except OSError:
            # Folder already exists, continue on
            pass
    
    def scrape_photos(self):
        # Setup directory to store photos
        scrape_dir = os.path.join(os.curdir,"facebook")
        photos_dir = os.path.join(scrape_dir,"photos")
        me_dir = os.path.join(photos_dir,"me")
    
        for d in [photos_dir, me_dir]:
            try:
                print "Creating %s directory...." % d
                os.mkdir(d)
            except OSError:
                # Folder already exists, continue on
                pass
        
        graph = self.api_request("/me/photos")
        
        try: 
            ps = graph["data"]
        except:
            ps = []
    
        pool = Pool(processes=25)
    
        print "Downloading %d photos of you...." % len(ps)
    
        for i, photo in enumerate(ps):
            purl = photo["source"]
            filename = os.path.join(me_dir, "myself_%s.jpg" % i)
            pool.apply_async(save_photo, [purl, filename, self.debug])
            
        albums = self.api_request("/me/albums")
        
        try:
            albums = albums["data"]
        except:
            albums = []
        
        for album in albums:
            try:
                albumd = slugify(album["name"])
                album_dir = os.path.join(photos_dir,albumd)
                print "Creating %s directory...." % album_dir
                os.mkdir(album_dir)
            except OSError:
                # Folder already exists, continue on
                pass
            
            photos = self.api_request("/%s/photos" % album["id"])
            try:
                photos = photos["data"]
            except:
                photos = []
                
            for i, photo in enumerate(photos):
                purl = photo["source"]
                filename = os.path.join(album_dir, "%s_%d.jpg" % (albumd, i))
                pool.apply_async(save_photo, [purl, filename, self.debug])
        
        pool.close()
        pool.join()
        
        

        
def main():
    try:
        opts, args = getopt.getopt(sys.argv[1:], "t:hd", ["token", "help", "debug"])
    except getopt.GetoptError, err:
        # print help information and exit:
        print str(err) # will print something like "option -a not recognized"
        usage()
        sys.exit(2)

    if not len(opts):
        usage()
        sys.exit(2)
        
    scraper = Scrapebook()

    for o, a in opts:
        if o in ("-t", "--token"):
            scraper.token = a
        if o in ("-d"):
            scraper.debug = True
        else:
            usage()
            
    if scraper.token:
        scraper.setup()
        scraper.scrape_photos()


{'fraction_non_alphanumeric': 0.095216031507008, 'fraction_numerical': 0.004633383528321557, 'mean_word_length': 2.112472963229993, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2677831', 'n_tokens_mistral': 2528, 'n_tokens_neox': 2429, 'n_words': 854}