<commit_before># -*- coding: utf-8 -*-
import harvester.image_harvest
harvester.image_harvest.link_is_to_image = lambda x,y: True
from harvester.image_harvest import ImageHarvester
from harvester.image_harvest import COUCHDB_VIEW, BUCKET_BASES

class ImageHarvesterOPLPreservica(ImageHarvester):
    def __init__(self, cdb=None,
                 url_couchdb=None,
                 couchdb_name=None,
                 couch_view=COUCHDB_VIEW,
                 bucket_bases=BUCKET_BASES,
                 object_auth=None,
                 get_if_object=False,
                 url_cache=None,
                 hash_cache=None,
                 harvested_object_cache=None,
                 auth_token=None):
        super(ImageHarvesterOPLPreservica, self).__init__(cdb=cdb,
                 url_couchdb=url_couchdb,
                 couchdb_name=couchdb_name,
                 couch_view=couch_view,
                 bucket_bases=bucket_bases,
                 object_auth=object_auth,
                 get_if_object=get_if_object,
                 url_cache=url_cache,
                 hash_cache=hash_cache,
                 harvested_object_cache=harvested_object_cache)
        self.auth_token = auth_token

    def stash_image(self, doc):
        url_image_base = doc.get('isShownBy', None)
        if url_image_base:
            doc['isShownBy'] = url_image_base +'&token={}'.format(self.auth_token)
        else:
            return None
        report =  super(ImageHarvesterOPLPreservica, self).stash_image(doc)
        doc['isShownBy'] = url_image_base
        return report

def main(collection_key=None,
         url_couchdb=None,
         object_auth=None,
         get_if_object=False,
         auth_token=None):<commit_msg>For OPL image harvest, if a text type don't grab image<commit_after># -*- coding: utf-8 -*-
import harvester.image_harvest
harvester.image_harvest.link_is_to_image = lambda x,y: True
from harvester.image_harvest import ImageHarvester
from harvester.image_harvest import COUCHDB_VIEW, BUCKET_BASES

class ImageHarvesterOPLPreservica(ImageHarvester):
    def __init__(self, cdb=None,
                 url_couchdb=None,
                 couchdb_name=None,
                 couch_view=COUCHDB_VIEW,
                 bucket_bases=BUCKET_BASES,
                 object_auth=None,
                 get_if_object=False,
                 url_cache=None,
                 hash_cache=None,
                 harvested_object_cache=None,
                 auth_token=None):
        super(ImageHarvesterOPLPreservica, self).__init__(cdb=cdb,
                 url_couchdb=url_couchdb,
                 couchdb_name=couchdb_name,
                 couch_view=couch_view,
                 bucket_bases=bucket_bases,
                 object_auth=object_auth,
                 get_if_object=get_if_object,
                 url_cache=url_cache,
                 hash_cache=hash_cache,
                 harvested_object_cache=harvested_object_cache)
        self.auth_token = auth_token

    def stash_image(self, doc):
        if doc['sourceResource']['type'] == 'text':
            return None
        url_image_base = doc.get('isShownBy', None)
        if url_image_base:
            doc['isShownBy'] = url_image_base +'&token={}'.format(self.auth_token)
        else:
            return None
        report =  super(ImageHarvesterOPLPreservica, self).stash_image(doc)
        doc['isShownBy'] = url_image_base
        return report

def main(collection_key=None,
         url_couchdb=None,
         object_auth=None,
         get_if_object=False,
         auth_token=None):
{'fraction_non_alphanumeric': 0.07602339181286549, 'fraction_numerical': 0.000556947925368978, 'mean_word_length': 2.131647776809067, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2635639', 'n_tokens_mistral': 1103, 'n_tokens_neox': 1095, 'n_words': 160}