<commit_before>#!/usr/bin/env python
#
# Util to download a full Flickr set.
#

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import argparse
import codecs
import errno
import locale
import logging
import os
import sys
import time

import flickr_api as Flickr
from flickr_api.flickrerrors import FlickrAPIError
from dateutil import parser
import yaml

from flickr_download.filename_handlers import get_filename_handler
from flickr_download.filename_handlers import get_filename_handler_help
from flickr_download.utils import get_full_path

CONFIG_FILE = "~/.flickr_download"
OAUTH_TOKEN_FILE = "~/.flickr_token"


def _init(key, secret, oauth):
    """
    Initialize API.

    @see: http://www.flickr.com/services/api/

    @param key: str, API key
    @param secret: str, API secret
    """
    Flickr.set_keys(key, secret)
    if not oauth:
        return True

    if os.path.exists(os.path.expanduser(OAUTH_TOKEN_FILE)):
        Flickr.set_auth_handler(os.path.expanduser(OAUTH_TOKEN_FILE))
        return True

    # Get new OAuth credentials
    auth = Flickr.auth.AuthHandler()  # creates the AuthHandler object
    perms = "read"  # set the required permissions
    url = auth.get_authorization_url(perms)
    print
    print("\nEnter the following url in a browser to authorize the application:")
    print(url)
    print("Copy and paste the <oauth_verifier> value from XML here and press return:")
    Flickr.set_auth_handler(auth)
    token = raw_input()
    auth.set_verifier(token)
    auth.save(os.path.expanduser(OAUTH_TOKEN_FILE))
    print("OAuth token was saved, re-run script to use it.")
    return False


def _load_defaults():
    """
    Load default parameters from config file

    @return: dict, default parameters
    """
    filename = os.path.expanduser(CONFIG_FILE)
    logging.debug('Loading configuration from {}'.format(filename))
    try:
        with open(filename, 'r') as cfile:
            vals = yaml.load(cfile.read())
            return vals
    except yaml.YAMLError as ex:
        logging.warning('Could not parse configuration file: {}'.format(ex))
    except IOError as ex:
        if ex.errno != errno.ENOENT:
            logging.warning('Could not open configuration file: {}'.format(ex))
        else:
            logging.debug('No config file')

    return {}


def download_set(set_id, get_filename, size_label=None):
    """
    Download the set with 'set_id' to the current directory.

    @param set_id: str, id of the photo set
    @param get_filename: Function, function that creates a filename for the photo
    @param size_label: str|None, size to download (or None for largest available)
    """
    pset = Flickr.Photoset(id=set_id)
    download_list(pset, pset.title, get_filename, size_label)


def download_list(pset, photos_title, get_filename, size_label):
    """
    Download all the photos in the given photo list

    @param pset: FlickrList, photo list to download
    @param photos_title: str, name of the photo list
    @param get_filename: Function, function that creates a filename for the photo
    @param size_label: str|None, size to download (or None for largest available)
    """
    photos = pset.getPhotos()
    pagenum = 2
    while True:
        try:
            if pagenum > photos.info.pages:
                break
            page = pset.getPhotos(page=pagenum)
            photos.extend(page)
            pagenum += 1
        except FlickrAPIError as ex:
            if ex.code == 1:
                break
            raise

    suffix = " ({})".format(size_label) if size_label else ""

    # we need to convert pathname separator to something else to create a valid directory
    dirname = photos_title.replace(os.sep, "_")
    if not os.path.exists(dirname):
        os.mkdir(dirname)

    for photo in photos:
        do_download_photo(dirname, pset, photo, size_label, suffix, get_filename)


def do_download_photo(dirname, pset, photo, size_label, suffix, get_filename):
    """
    Handle the downloading of a single photo

    @param dirname: str, directory to download to
    @param pset: FlickrList, photo list to download
    @param photo: FlickrPhoto, photo to download
    @param size_label: str|None, size to download (or None for largest available)
    @param suffix: str|None, optional suffix to add to file name
    @param get_filename: Function, function that creates a filename for the photo
    """
    fname = get_full_path(dirname, get_filename(pset, photo, suffix))

    if 'video' in photo.getInfo():
        if 'HD MP4' in photo.getSizes():
            photo_size_label = 'HD MP4'
        else:
            # Fall back for old 'short videos'
            photo_size_label = 'Site MP4'
        fname = fname + '.mp4'
    else:
        photo_size_label = size_label
        fname = fname + '.jpg'

    if os.path.exists(fname):
        # TODO: Ideally we should check for file size / md5 here
        # to handle failed downloads.
        print('Skipping {0}, as it exists already'.format(fname))
        return

    print('Saving: {} ({})'.format(fname, photo.getPageUrl()))
    try:
        photo.save(fname, photo_size_label)
    except IOError, ex:
        logging.warning('IO error saving photo: {}'.format(ex.strerror))
        return

    # Set file times to when the photo was taken
    info = photo.getInfo()
    taken = parser.parse(info['taken'])
    taken_unix = time.mktime(taken.timetuple())
    os.utime(fname, (taken_unix, taken_unix))


def download_photo(photo_id, get_filename, size_label):
    """
    Download one photo

    @param photo_id: str, id of the photo
    @param get_filename: Function, function that creates a filename for the photo
    @param size_label: str|None, size to download (or None for largest available)
    """
    photo = Flickr.Photo(id=photo_id)
    suffix = " ({})".format(size_label) if size_label else ""
    do_download_photo(".", None, photo, size_label, suffix, get_filename)


def download_user(username, get_filename, size_label):
    """
    Download all the sets owned by the given user.

    @param username: str, username
    @param get_filename: Function, function that creates a filename for the photo
    @param size_label: str|None, size to download (or None for largest available)
    """
    user = Flickr.Person.findByUserName(username)
    photosets = user.getPhotosets()
    for photoset in photosets:
        download_set(photoset.id, get_filename, size_label)


def download_user_photos(username, get_filename, size_label):
    """
    Download all the photos owned by the given user.

    @param username: str, username
    @param get_filename: Function, function that creates a filename for the photo
    @param size_label: str|None, size to download (or None for largest available)
    """
    user = Flickr.Person.findByUserName(username)
    download_list(user, username, get_filename, size_label)


def print_sets(username):
    """
    Print all sets for the given user

    @param username: str,
    """
    user = Flickr.Person.findByUserName(username)
    photosets = user.getPhotosets()
    for photo in photosets:
        print('{0} - {1}'.format(photo.id, photo.title))


def main():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawTextHelpFormatter,
        description='Downloads one or more Flickr photo sets.\n'
        '\n'
        'To use it you need to get your own Flickr API key here:\n'
        'https://www.flickr.com/services/api/misc.api_keys.html\n'
        '\n'
        'For more information see:\n'
        'https://github.com/beaufour/flickr-download\n'
        '\n'
        'You can store argument defaults in ' + CONFIG_FILE + '. API keys for example:\n'
        '  api_key: .....\n'
        '  api_secret: ...\n'
        ,
        epilog='examples:\n'<commit_msg>pep8 fix last commit<commit_after>#!/usr/bin/env python
#
# Util to download a full Flickr set.
#

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import argparse
import codecs
import errno
import locale
import logging
import os
import sys
import time

import flickr_api as Flickr
from flickr_api.flickrerrors import FlickrAPIError
from dateutil import parser
import yaml

from flickr_download.filename_handlers import get_filename_handler
from flickr_download.filename_handlers import get_filename_handler_help
from flickr_download.utils import get_full_path

CONFIG_FILE = "~/.flickr_download"
OAUTH_TOKEN_FILE = "~/.flickr_token"


def _init(key, secret, oauth):
    """
    Initialize API.

    @see: http://www.flickr.com/services/api/

    @param key: str, API key
    @param secret: str, API secret
    """
    Flickr.set_keys(key, secret)
    if not oauth:
        return True

    if os.path.exists(os.path.expanduser(OAUTH_TOKEN_FILE)):
        Flickr.set_auth_handler(os.path.expanduser(OAUTH_TOKEN_FILE))
        return True

    # Get new OAuth credentials
    auth = Flickr.auth.AuthHandler()  # creates the AuthHandler object
    perms = "read"  # set the required permissions
    url = auth.get_authorization_url(perms)
    print
    print("\nEnter the following url in a browser to authorize the application:")
    print(url)
    print("Copy and paste the <oauth_verifier> value from XML here and press return:")
    Flickr.set_auth_handler(auth)
    token = raw_input()
    auth.set_verifier(token)
    auth.save(os.path.expanduser(OAUTH_TOKEN_FILE))
    print("OAuth token was saved, re-run script to use it.")
    return False


def _load_defaults():
    """
    Load default parameters from config file

    @return: dict, default parameters
    """
    filename = os.path.expanduser(CONFIG_FILE)
    logging.debug('Loading configuration from {}'.format(filename))
    try:
        with open(filename, 'r') as cfile:
            vals = yaml.load(cfile.read())
            return vals
    except yaml.YAMLError as ex:
        logging.warning('Could not parse configuration file: {}'.format(ex))
    except IOError as ex:
        if ex.errno != errno.ENOENT:
            logging.warning('Could not open configuration file: {}'.format(ex))
        else:
            logging.debug('No config file')

    return {}


def download_set(set_id, get_filename, size_label=None):
    """
    Download the set with 'set_id' to the current directory.

    @param set_id: str, id of the photo set
    @param get_filename: Function, function that creates a filename for the photo
    @param size_label: str|None, size to download (or None for largest available)
    """
    pset = Flickr.Photoset(id=set_id)
    download_list(pset, pset.title, get_filename, size_label)


def download_list(pset, photos_title, get_filename, size_label):
    """
    Download all the photos in the given photo list

    @param pset: FlickrList, photo list to download
    @param photos_title: str, name of the photo list
    @param get_filename: Function, function that creates a filename for the photo
    @param size_label: str|None, size to download (or None for largest available)
    """
    photos = pset.getPhotos()
    pagenum = 2
    while True:
        try:
            if pagenum > photos.info.pages:
                break
            page = pset.getPhotos(page=pagenum)
            photos.extend(page)
            pagenum += 1
        except FlickrAPIError as ex:
            if ex.code == 1:
                break
            raise

    suffix = " ({})".format(size_label) if size_label else ""

    # we need to convert pathname separator to something else to create a valid directory
    dirname = photos_title.replace(os.sep, "_")
    if not os.path.exists(dirname):
        os.mkdir(dirname)

    for photo in photos:
        do_download_photo(dirname, pset, photo, size_label, suffix, get_filename)


def do_download_photo(dirname, pset, photo, size_label, suffix, get_filename):
    """
    Handle the downloading of a single photo

    @param dirname: str, directory to download to
    @param pset: FlickrList, photo list to download
    @param photo: FlickrPhoto, photo to download
    @param size_label: str|None, size to download (or None for largest available)
    @param suffix: str|None, optional suffix to add to file name
    @param get_filename: Function, function that creates a filename for the photo
    """
    fname = get_full_path(dirname, get_filename(pset, photo, suffix))

    if 'video' in photo.getInfo():
        if 'HD MP4' in photo.getSizes():
            photo_size_label = 'HD MP4'
        else:
            # Fall back for old 'short videos'
            photo_size_label = 'Site MP4'
        fname = fname + '.mp4'
    else:
        photo_size_label = size_label
        fname = fname + '.jpg'

    if os.path.exists(fname):
        # TODO: Ideally we should check for file size / md5 here
        # to handle failed downloads.
        print('Skipping {0}, as it exists already'.format(fname))
        return

    print('Saving: {} ({})'.format(fname, photo.getPageUrl()))
    try:
        photo.save(fname, photo_size_label)
    except IOError, ex:
        logging.warning('IO error saving photo: {}'.format(ex.strerror))
        return

    # Set file times to when the photo was taken
    info = photo.getInfo()
    taken = parser.parse(info['taken'])
    taken_unix = time.mktime(taken.timetuple())
    os.utime(fname, (taken_unix, taken_unix))


def download_photo(photo_id, get_filename, size_label):
    """
    Download one photo

    @param photo_id: str, id of the photo
    @param get_filename: Function, function that creates a filename for the photo
    @param size_label: str|None, size to download (or None for largest available)
    """
    photo = Flickr.Photo(id=photo_id)
    suffix = " ({})".format(size_label) if size_label else ""
    do_download_photo(".", None, photo, size_label, suffix, get_filename)


def download_user(username, get_filename, size_label):
    """
    Download all the sets owned by the given user.

    @param username: str, username
    @param get_filename: Function, function that creates a filename for the photo
    @param size_label: str|None, size to download (or None for largest available)
    """
    user = Flickr.Person.findByUserName(username)
    photosets = user.getPhotosets()
    for photoset in photosets:
        download_set(photoset.id, get_filename, size_label)


def download_user_photos(username, get_filename, size_label):
    """
    Download all the photos owned by the given user.

    @param username: str, username
    @param get_filename: Function, function that creates a filename for the photo
    @param size_label: str|None, size to download (or None for largest available)
    """
    user = Flickr.Person.findByUserName(username)
    download_list(user, username, get_filename, size_label)


def print_sets(username):
    """
    Print all sets for the given user

    @param username: str,
    """
    user = Flickr.Person.findByUserName(username)
    photosets = user.getPhotosets()
    for photo in photosets:
        print('{0} - {1}'.format(photo.id, photo.title))


def main():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawTextHelpFormatter,
        description='Downloads one or more Flickr photo sets.\n'
        '\n'
        'To use it you need to get your own Flickr API key here:\n'
        'https://www.flickr.com/services/api/misc.api_keys.html\n'
        '\n'
        'For more information see:\n'
        'https://github.com/beaufour/flickr-download\n'
        '\n'
        'You can store argument defaults in ' + CONFIG_FILE + '. API keys for example:\n'
        '  api_key: .....\n'
        '  api_secret: ...\n',
        epilog='examples:\n'
{'fraction_non_alphanumeric': 0.09411764705882353, 'fraction_numerical': 0.001454775458570525, 'mean_word_length': 3.020086448004068, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 70, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '798866', 'n_tokens_mistral': 4815, 'n_tokens_neox': 4646, 'n_words': 1504}