<commit_before># harvest images for the given collection
# by this point the isShownBy value for the doc should be filled in.
# this should point at the highest fidelity object file available
# from the source.
# use brian's content md5s3stash to store the resulting image.

import os
import sys
import datetime
import time
import couchdb
import requests
import md5s3stash
from harvester.couchdb_init import get_couchdb
from harvester.config import config
from redis import Redis
import redis_collections

BUCKET_BASE = os.environ.get('S3_BUCKET_IMAGE_BASE', 'ucldc')
COUCHDB_VIEW = 'all_provider_docs/by_provider_name'
URL_OAC_CONTENT_BASE = os.environ.get('URL_OAC_CONTENT_BASE',
                                      'http://content.cdlib.org')

def link_is_to_image(url):
    '''Check if the link points to an image content type.
    Return True or False accordingly
    '''
    response = requests.head(url, allow_redirects=True)
    if response.status_code != 200:
        return False
    content_type = response.headers.get('content-type', None)
    if not content_type:
        return False
    reg_type = content_type.split('/', 1)[0].lower()
    return reg_type == 'image'

class ImageHarvester(object):
    '''Useful to cache couchdb, authentication info and such'''
    def __init__(self, cdb=None,
                 url_couchdb=None,
                 couchdb_name=None,
                 couch_view=COUCHDB_VIEW,
                 bucket_base=BUCKET_BASE,
                 object_auth=None,
                 no_get_if_object=False):
        if cdb:
            self._couchdb = cdb
        else:
            self._couchdb = get_couchdb(url=url_couchdb,
                                        dbname=couchdb_name)
        self._bucket_base = bucket_base
        self._view = couch_view
        # auth is a tuple of username, password
        self._auth = object_auth
        self.no_get_if_object = no_get_if_object # if object field exists, try to get
        
        self._config = config()
        self._redis = Redis(host=self._config['redis_host'],
                            port=self._config['redis_port'],
                            password=self._config['redis_password'],
                            socket_connect_timeout=self._config['redis_connect_timeout'])
        self._url_cache = redis_collections.Dict(key='ucldc-image-url-cache',
                redis=self._redis)
        self._hash_cache = redis_collections.Dict(key='ucldc-image-hash-cache',
                redis=self._redis)

    # Need to make each download a separate job.
    def stash_image(self, doc):
        '''Stash the images in s3, using md5s3stash
        Return md5s3stash report if image found
        If link is not an image type, don't stash & return None
        '''
        try:
            url_image = doc['isShownBy']
            if not url_image:
                raise ValueError("isShownBy empty for {0}".format(doc['_id']))
        except KeyError, e:
            raise KeyError("isShownBy missing for {0}".format(doc['_id']))
        if isinstance(url_image, list): # need to fix marc map_is_shown_at
            url_image = url_image[0]
        # for some OAC objects, the reference image is not a url but a path.
        if 'http' != url_image[:4] and 'https' != url_image[:5]:
            # not a URL....
            if 'ark:' in url_image:
                url_image = '/'.join((URL_OAC_CONTENT_BASE, url_image))
            else:
                print >> sys.stderr, 'Link not http URL for {} - {}'.format(
                                      doc['_id'], url_image)
                return None
        if link_is_to_image(url_image):
            return md5s3stash.md5s3stash(url_image,
                                         bucket_base=self._bucket_base,
                                         url_auth=self._auth,
                                         url_cache=self._url_cache,
                                         hash_cache=self._hash_cache)
        else:
            print >> sys.stderr, 'Not an image for {} - {}'.format(
                                      doc['_id'], url_image)
            return None

    def update_doc_object(self, doc, report):
        '''Update the object field to point to an s3 bucket'''
        doc['object'] = report.md5
        doc['object_dimensions'] = report.dimensions
        self._couchdb.save(doc)
        return doc['object']

    def harvest_image_for_doc(self, doc):
        '''Try to harvest an image for a couchdb doc'''
        report = None
        if self.no_get_if_object:
            if doc.get('object', None):
                print >> sys.stderr, 'Skipping {}, has object field'.format(doc['_id'])
                return
        try:
            report = self.stash_image(doc)
            if report != None:
                obj_val = self.update_doc_object(doc, report)
        except KeyError, e:
            if 'isShownBy' in e.message:
                 print >> sys.stderr, e
            else:
                raise e
        except ValueError, e:
            if 'isShownBy' in e.message:
                 print >> sys.stderr, e
            else:
                raise e
        except IOError, e:
             print >> sys.stderr, e
        return report

    def by_list_of_doc_ids(self, doc_ids):
        '''For a list of ids, harvest images'''
        for doc_id in doc_ids:
            doc = self._couchdb[doc_id]
            dt_start = dt_end = datetime.datetime.now()
            report = self.harvest_image_for_doc(doc)
            dt_end = datetime.datetime.now()
            time.sleep((dt_end-dt_start).total_seconds())

    def by_collection(self, collection_key=None):
        '''If collection_key is none, trying to grab all of the images. (Not
        recommended)
        '''
        if collection_key:
            v = self._couchdb.view(self._view, include_docs='true',
                                   key=collection_key)
        else:
            v = self._couchdb.view(self._view, include_docs='true')
        doc_ids = []
        for r in v:
            dt_start = dt_end = datetime.datetime.now()
            report = self.harvest_image_for_doc(r.doc)
            doc_ids.append(r.doc['_id'])
            dt_end = datetime.datetime.now()
            time.sleep((dt_end-dt_start).total_seconds())
        return doc_ids

def harvest_image_for_doc(doc_id,
        url_couchdb=None,<commit_msg>Use pager when grabbing view.<commit_after># harvest images for the given collection
# by this point the isShownBy value for the doc should be filled in.
# this should point at the highest fidelity object file available
# from the source.
# use brian's content md5s3stash to store the resulting image.

import os
import sys
import datetime
import time
import couchdb
import requests
import md5s3stash
from harvester.couchdb_init import get_couchdb
from harvester.config import config
from redis import Redis
import redis_collections
from harvester.couchdb_pager import couchdb_pager

BUCKET_BASE = os.environ.get('S3_BUCKET_IMAGE_BASE', 'ucldc')
COUCHDB_VIEW = 'all_provider_docs/by_provider_name'
URL_OAC_CONTENT_BASE = os.environ.get('URL_OAC_CONTENT_BASE',
                                      'http://content.cdlib.org')

def link_is_to_image(url):
    '''Check if the link points to an image content type.
    Return True or False accordingly
    '''
    response = requests.head(url, allow_redirects=True)
    if response.status_code != 200:
        return False
    content_type = response.headers.get('content-type', None)
    if not content_type:
        return False
    reg_type = content_type.split('/', 1)[0].lower()
    return reg_type == 'image'

class ImageHarvester(object):
    '''Useful to cache couchdb, authentication info and such'''
    def __init__(self, cdb=None,
                 url_couchdb=None,
                 couchdb_name=None,
                 couch_view=COUCHDB_VIEW,
                 bucket_base=BUCKET_BASE,
                 object_auth=None,
                 no_get_if_object=False):
        if cdb:
            self._couchdb = cdb
        else:
            self._couchdb = get_couchdb(url=url_couchdb,
                                        dbname=couchdb_name)
        self._bucket_base = bucket_base
        self._view = couch_view
        # auth is a tuple of username, password
        self._auth = object_auth
        self.no_get_if_object = no_get_if_object # if object field exists, try to get
        
        self._config = config()
        self._redis = Redis(host=self._config['redis_host'],
                            port=self._config['redis_port'],
                            password=self._config['redis_password'],
                            socket_connect_timeout=self._config['redis_connect_timeout'])
        self._url_cache = redis_collections.Dict(key='ucldc-image-url-cache',
                redis=self._redis)
        self._hash_cache = redis_collections.Dict(key='ucldc-image-hash-cache',
                redis=self._redis)

    # Need to make each download a separate job.
    def stash_image(self, doc):
        '''Stash the images in s3, using md5s3stash
        Return md5s3stash report if image found
        If link is not an image type, don't stash & return None
        '''
        try:
            url_image = doc['isShownBy']
            if not url_image:
                raise ValueError("isShownBy empty for {0}".format(doc['_id']))
        except KeyError, e:
            raise KeyError("isShownBy missing for {0}".format(doc['_id']))
        if isinstance(url_image, list): # need to fix marc map_is_shown_at
            url_image = url_image[0]
        # for some OAC objects, the reference image is not a url but a path.
        if 'http' != url_image[:4] and 'https' != url_image[:5]:
            # not a URL....
            if 'ark:' in url_image:
                url_image = '/'.join((URL_OAC_CONTENT_BASE, url_image))
            else:
                print >> sys.stderr, 'Link not http URL for {} - {}'.format(
                                      doc['_id'], url_image)
                return None
        if link_is_to_image(url_image):
            return md5s3stash.md5s3stash(url_image,
                                         bucket_base=self._bucket_base,
                                         url_auth=self._auth,
                                         url_cache=self._url_cache,
                                         hash_cache=self._hash_cache)
        else:
            print >> sys.stderr, 'Not an image for {} - {}'.format(
                                      doc['_id'], url_image)
            return None

    def update_doc_object(self, doc, report):
        '''Update the object field to point to an s3 bucket'''
        doc['object'] = report.md5
        doc['object_dimensions'] = report.dimensions
        self._couchdb.save(doc)
        return doc['object']

    def harvest_image_for_doc(self, doc):
        '''Try to harvest an image for a couchdb doc'''
        report = None
        if self.no_get_if_object:
            if doc.get('object', None):
                print >> sys.stderr, 'Skipping {}, has object field'.format(doc['_id'])
                return
        try:
            report = self.stash_image(doc)
            if report != None:
                obj_val = self.update_doc_object(doc, report)
        except KeyError, e:
            if 'isShownBy' in e.message:
                 print >> sys.stderr, e
            else:
                raise e
        except ValueError, e:
            if 'isShownBy' in e.message:
                 print >> sys.stderr, e
            else:
                raise e
        except IOError, e:
             print >> sys.stderr, e
        return report

    def by_list_of_doc_ids(self, doc_ids):
        '''For a list of ids, harvest images'''
        for doc_id in doc_ids:
            doc = self._couchdb[doc_id]
            dt_start = dt_end = datetime.datetime.now()
            report = self.harvest_image_for_doc(doc)
            dt_end = datetime.datetime.now()
            time.sleep((dt_end-dt_start).total_seconds())

    def by_collection(self, collection_key=None):
        '''If collection_key is none, trying to grab all of the images. (Not
        recommended)
        '''
        if collection_key:
            v = couchdb_pager(self._couchdb, view_name=self._view,
                    startkey=[collection_key],
                    endkey=[collection_key, {}],
                    include_docs='true')
        else:
            #use _all_docs view
            v = couchdb_pager(self._couchdb, include_docs='true')
        doc_ids = []
        for r in v:
            dt_start = dt_end = datetime.datetime.now()
            report = self.harvest_image_for_doc(r.doc)
            doc_ids.append(r.doc['_id'])
            dt_end = datetime.datetime.now()
            time.sleep((dt_end-dt_start).total_seconds())
        return doc_ids

def harvest_image_for_doc(doc_id,
        url_couchdb=None,
{'fraction_non_alphanumeric': 0.09095109742010012, 'fraction_numerical': 0.004004620716211013, 'mean_word_length': 1.858463570327977, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 27, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3595854', 'n_tokens_mistral': 3739, 'n_tokens_neox': 3692, 'n_words': 942}