<commit_before>#! /usr/bin/env python

import sys
import logging

from optparse import OptionParser
from pymongo.connection import Connection

import flickr
import settings

try:
    from mongolog.handlers import MongoHandler
except ImportError:
    print >>sys.stderr, 'Please install mongolog. This will allow centralized logging.'
    sys.exit(1)

def main():
    log = setup_logging()
    log.setLevel(logging.DEBUG)
    log.info('Starting Flickr photo fetch tool');

    opt, args = parse_args()
    f = flickr.Flickr()

    try:
        num = int(opt.num)
    except (ValueError, TypeError):
        num = 5

    log.info("Query max result size: %d" % num)

    results = None
    if opt.query is not None:
        log.info('Searching flickr: "%s"' % opt.query)
        results = f.search_photos(opt.query, num)
    elif opt.interesting:
        log.info('Loading interesting pictures from flickr.')
        results = f.interesting_photos(num, opt.date)
    elif opt.recent:
        log.info('Loading recent photos from flickr.')
        results = f.recent_photos(num)
    else:
        print 'Usage: ./flickr_fetch.py <options> or --help for info'
        return 2

    if results is not None:
        store_photos(f, results, log)
    else:
        log.info('No photos found.')

def setup_logging():
    """ Setup centralized logging

    All log messages will end-up in capped mongo collection
    """
    log = logging.getLogger('flickr')
    host = 'localhost'
    try:
        import settings
        host = settings.MONGO['host']
    except ImportError:
        print >>sys.stderr, 'Settings file is not available. Using localhost.'

    log.addHandler(MongoHandler.to(db='logging', collection='tools', host=host))
    return log       

def store_photos(f, results, log):
    conn = Connection(settings.MONGO['host'], settings.MONGO['port'])
    collection = conn[settings.MONGO['db']]['photos']

    for id, title in results:
        if collection.find({'id':id}).count():
            log.info('Photo with ID:%s is already stored.' % id)
            continue

        photo_data = {
            'id' : id,
            'title' : title, 
            'urls' : f.get_photo_urls(id),
            'exif' : list(f.get_exif(id))
        }
        info = f.get_photo_info(id)
        if info is not None:
            for k,v in info.items():
                photo_data[k] = v

        if 'Medium' in photo_data['urls']:
            log.info("Saving photo: ID:%s Title:%s Url:%s" %  (id, title, photo_data['urls']['Medium']))<commit_msg>small refactorings<commit_after>#! /usr/bin/env python

import sys
import logging

from optparse import OptionParser
from pymongo.connection import Connection

import flickr
import settings

try:
    from mongolog.handlers import MongoHandler
except ImportError:
    print >>sys.stderr, 'Please install mongolog. This will allow centralized logging.'
    sys.exit(1)

def main():
    log = setup_logging(logging.DEBUG)
    log.info('Starting Flickr photo fetch tool');

    opt, args = parse_args()
    f = flickr.Flickr()

    try:
        num = int(opt.num)
    except (ValueError, TypeError):
        num = 5

    log.info("Query max result size: %d" % num)

    results = None
    if opt.query is not None:
        log.info('Searching flickr: "%s"' % opt.query)
        results = f.search_photos(opt.query, num)
    elif opt.interesting:
        log.info('Loading interesting pictures from flickr.')
        results = f.interesting_photos(num, opt.date)
    elif opt.recent:
        log.info('Loading recent photos from flickr.')
        results = f.recent_photos(num)
    else:
        print 'Usage: ./flickr_fetch.py <options> or --help for info'
        return 2

    if results is not None:
        store_photos(f, results, log)
    else:
        log.info('No photos found.')

def setup_logging(level=logging.DEBUG):
    """ Setup centralized logging

    All log messages will end-up in capped mongo collection
    """
    log = logging.getLogger('flickr')
    host = 'localhost'
    try:
        import settings
        host = settings.MONGO['host']

    except ImportError:
        print >>sys.stderr, 'Settings file is not available. Using localhost.'

    log.addHandler(MongoHandler.to(db='logging', collection='tools', host=host))
    log.setLevel(level)

    return log 

def store_photos(f, results, log):
    conn = Connection(settings.MONGO['host'], settings.MONGO['port'])
    collection = conn[settings.MONGO['db']]['photos']

    for id, title in results:
        if collection.find({'id':id}).count():
            log.info('Photo with ID:%s is already stored.' % id)
            continue

        photo_data = {
            'id' : id,
            'title' : title, 
            'urls' : f.get_photo_urls(id),
            'exif' : list(f.get_exif(id))
        }
        info = f.get_photo_info(id)
        if info is not None:
            for k,v in info.items():
                photo_data[k] = v

        if 'Medium' in photo_data['urls']:
            log.info("Saving photo: ID:%s Title:%s Url:%s" %  (id, title, photo_data['urls']['Medium']))
{'fraction_non_alphanumeric': 0.12135447249951067, 'fraction_numerical': 0.0011743981209630064, 'mean_word_length': 2.507206588881263, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5851615', 'n_tokens_mistral': 1602, 'n_tokens_neox': 1524, 'n_words': 438}