<commit_before># harvest images for the given collection
# by this point the isShownBy value for the doc should be filled in.
# this should point at the highest fidelity object file available
# from the source.
# use brian's content md5s3stash to store the resulting image.

import os
import sys
import datetime
import time
import urlparse
import couchdb
import requests
import md5s3stash
from harvester.couchdb_init import get_couchdb
from harvester.config import config
from redis import Redis
import redis_collections
from harvester.couchdb_pager import couchdb_pager

BUCKET_BASE = os.environ.get('S3_BUCKET_IMAGE_BASE', 'ucldc')
COUCHDB_VIEW = 'all_provider_docs/by_provider_name'
URL_OAC_CONTENT_BASE = os.environ.get('URL_OAC_CONTENT_BASE',
                                      'http://content.cdlib.org')

def link_is_to_image(url):
    '''Check if the link points to an image content type.
    Return True or False accordingly
    '''
    response = requests.head(url, allow_redirects=True,
            auth=requests.auth.HTTPBasicAuth(self.__auth))
    if response.status_code != 200:
        return False
    content_type = response.headers.get('content-type', None)
    if not content_type:
        return False
    reg_type = content_type.split('/', 1)[0].lower()
    return reg_type == 'image'

class ImageHarvester(object):
    '''Useful to cache couchdb, authentication info and such'''
    def __init__(self, cdb=None,
                 url_couchdb=None,
                 couchdb_name=None,
                 couch_view=COUCHDB_VIEW,
                 bucket_base=BUCKET_BASE,
                 object_auth=None,
                 no_get_if_object=False):
        if cdb:
            self._couchdb = cdb
        else:
            self._couchdb = get_couchdb(url=url_couchdb,
                                        dbname=couchdb_name)
        self._bucket_base = bucket_base<commit_msg>use shortcut for auth param<commit_after># harvest images for the given collection
# by this point the isShownBy value for the doc should be filled in.
# this should point at the highest fidelity object file available
# from the source.
# use brian's content md5s3stash to store the resulting image.

import os
import sys
import datetime
import time
import urlparse
import couchdb
import requests
import md5s3stash
from harvester.couchdb_init import get_couchdb
from harvester.config import config
from redis import Redis
import redis_collections
from harvester.couchdb_pager import couchdb_pager

BUCKET_BASE = os.environ.get('S3_BUCKET_IMAGE_BASE', 'ucldc')
COUCHDB_VIEW = 'all_provider_docs/by_provider_name'
URL_OAC_CONTENT_BASE = os.environ.get('URL_OAC_CONTENT_BASE',
                                      'http://content.cdlib.org')

def link_is_to_image(url):
    '''Check if the link points to an image content type.
    Return True or False accordingly
    '''
    response = requests.head(url, allow_redirects=True,
            auth=self.__auth)
    if response.status_code != 200:
        return False
    content_type = response.headers.get('content-type', None)
    if not content_type:
        return False
    reg_type = content_type.split('/', 1)[0].lower()
    return reg_type == 'image'

class ImageHarvester(object):
    '''Useful to cache couchdb, authentication info and such'''
    def __init__(self, cdb=None,
                 url_couchdb=None,
                 couchdb_name=None,
                 couch_view=COUCHDB_VIEW,
                 bucket_base=BUCKET_BASE,
                 object_auth=None,
                 no_get_if_object=False):
        if cdb:
            self._couchdb = cdb
        else:
            self._couchdb = get_couchdb(url=url_couchdb,
                                        dbname=couchdb_name)
        self._bucket_base = bucket_base
{'fraction_non_alphanumeric': 0.07186261558784676, 'fraction_numerical': 0.005284015852047556, 'mean_word_length': 2.759682224428997, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4522718', 'n_tokens_mistral': 1130, 'n_tokens_neox': 1118, 'n_words': 309}