<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import urllib
import argparse
import re
import hashlib
from collections import defaultdict
import requests
import boto
from solr import Solr, SolrException
from harvester.couchdb_init import get_couchdb
from facet_decade import facet_decade
import datetime

S3_BUCKET = 'solr.ucldc'

RE_ARK_FINDER = re.compile('(ark:/\d\d\d\d\d/[^/|\s]*)')
RE_ALPHANUMSPACE = re.compile(r'[^0-9A-Za-z\s]*') #\W include "_" as does A-z

COUCHDOC_TO_SOLR_MAPPING = {
    '_id'       : lambda d: {'harvest_id_s': d['_id']},
    'object'   : lambda d: {'reference_image_md5': d['object']},
    'isShownAt': lambda d: {'url_item': d['isShownAt']},
}

class UTCtz(datetime.tzinfo):
    def utcoffset(self, dt):
        return datetime.timedelta(hours=0)
    def tzname(self, dt):
        return 'GMT'
    def dst(self, dt):
        return timedelta(0)
UTC = UTCtz()

def make_datetime(dstring):
    '''make a datetime from a valid date string
    Right now formats are YYYY or YYYY-MM-DD
    '''
    dt = None
    if not dstring:
        return dt
    try:
        dint = int(dstring)
        dt = datetime.datetime(dint, 1, 1)
    except ValueError:
        pass
    except TypeError, e:
        print >> sys.stderr, 'Date type err DATA:{} ERROR:{}'.format(dstring, e)
    try:
        strfmt = '%Y-%m-%d'
        dt = datetime.datetime.strptime(dstring, strfmt)
    except ValueError:
        pass
    except TypeError, e:
        print >> sys.stderr, 'Date type err in strptime:{} {}'.format(dstring, e)
    # add UTC as timezone, solrpy looks for tzinfo
    if dt:
        dt = datetime.datetime(dt.year, dt.month, dt.day, tzinfo=UTC)
    return dt

def get_dates_from_date_obj(date_obj):
    '''Return list of display date, start dates, end dates'''
    if isinstance(date_obj, dict):
        date_start = make_datetime(date_obj.get('begin', None))
        date_end = make_datetime(date_obj.get('end', None))
        dates = (date_obj.get('displayDate', None),
                date_start,
                date_end
                )
        return dates
    elif isinstance(date_obj, basestring):
        return date_obj, None, None
    else:
        return None, None, None

def map_date(d):
    date_map = {}
    date_source = d.get('date', None)
    dates = []
    start_date = end_date = None
    dates_start = []
    dates_end = []
    #print >> sys.stderr, 'date_source:{}'.format(date_source)
    if date_source:
        if isinstance(date_source, dict):
            try:
                displayDate, dt_start, dt_end = get_dates_from_date_obj(date_source)
                dates.append(displayDate)
                if dt_start:
                     dates_start.append(dt_start)
                if dt_end:
                    dates_end.append(dt_end)
            except KeyError:
                pass
        else: #should be list
            for dt in date_source:
                displayDate, dt_start, dt_end = get_dates_from_date_obj(dt)
                dates.append(displayDate)
                if dt_start:
                     dates_start.append(dt_start)
                if dt_end:
                    dates_end.append(dt_end)

    date_map['date'] = dates

    dates_start = sorted(dates_start)
    dates_end = sorted(dates_end)
    if len(dates_start):
    	start_date = dates_start[0]
    if len(dates_end):
    	end_date = dates_end[0]
    # fill in start_date == end_date if only one exists
    start_date = end_date if not start_date else start_date
    end_date = start_date if not end_date else end_date
    if start_date:
        #add timezone as UTC, for solrpy (uses astimzone())
        date_map['sort_date_start'] = start_date
        date_map['sort_date_end'] = end_date

    return date_map
        

COUCHDOC_SRC_RESOURCE_TO_SOLR_MAPPING = {
    'alternativeTitle'   : lambda d: {'alternative_title': d.get('alternativeTitle', None)},
    'contributor' : lambda d: {'contributor': d.get('contributor', None)},
    'coverage'    : lambda d: {'coverage': d.get('coverage', None)},
    'spatial'     : lambda d: {'coverage': [c['text'] if (isinstance(c, dict)
        and 'text' in c)  else c for c in d['spatial']]},
    'creator'     : lambda d: {'creator': d.get('creator', None)},
    'date'        : lambda d:  map_date(d),
    'description' : lambda d: {'description': [ds for ds in d['description']]},
    'extent'      : lambda d: {'extent': d.get('extent', None)},
    'format'      : lambda d: {'format': d.get('format', None)},
    'genre'       : lambda d: {'genre': d.get('genre', None)},
    'identifier'  : lambda d: {'identifier': d.get('identifier', None)},
    'language'    : lambda d: {'language': [l.get('iso639_3', l.get('name', None)) if isinstance(l, dict) else l for l in d['language']]},
    'publisher'   : lambda d: {'publisher': d.get('publisher', None)},
    'relation'    : lambda d: {'relation': d.get('relation', None)},
    'rights'      : lambda d: {'rights': d.get('rights', None)},
    'subject'     : lambda d: {'subject': [s['name'] if isinstance(s, dict) else s for s in d['subject']]},
    'temporalCoverage'    : lambda d: {'temporal': d.get('temporalCoverage', None)},
    'title'       : lambda d: {'title': d.get('title', None)},
    'type'        : lambda d: {'type': d.get('type', None)},
}

COUCHDOC_ORIGINAL_RECORD_TO_SOLR_MAPPING = {
    'location'        : lambda d: {'location': d.get('location', None)},
    'provenance'      : lambda d: {'provenance': d.get('provenance', None)},
    'dateCopyrighted' : lambda d: {'rights_date': d.get('dateCopyrighted')},
    'rightsHolder'    : lambda d: {'rights_holder': d.get('rightsHolder')},
    'rightsNote'      : lambda d: {'rights_note': d.get('rightsNote')},
    'source'          : lambda d: {'source': d.get('source')},
    'structmap_text'  : lambda d: {'structmap_text': d.get('structmap_text')},
    'structmap_url'   : lambda d: {'structmap_url': d.get('structmap_url')},
    'transcription'   : lambda d: {'transcription': d.get('transcription')},
}

def find_ark_in_identifiers(doc):
    identifiers = doc['sourceResource'].get('identifier', None)
    if identifiers:
        for identifier in identifiers:
            match = RE_ARK_FINDER.search(identifier)
            if match:
                return match.group(0)
    return None

def uuid_if_nuxeo(doc):
    collection = doc['originalRecord']['collection'][0]
    harvest_type = collection['harvest_type']
    if harvest_type == 'NUX':
        return doc['originalRecord'].get('uid', None)
    return None

def ucsd_ark(doc):
    #is this UCSD?
    campus = None
    ark = None
    collection = doc['originalRecord']['collection'][0]<commit_msg>intermediate data reporting version, before dejson working<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import urllib
import argparse
import re
import hashlib
import json
from collections import defaultdict
import requests
import boto
from solr import Solr, SolrException
from harvester.couchdb_init import get_couchdb
from facet_decade import facet_decade
import datetime

reload(sys)
sys.setdefaultencoding('utf8')
 
S3_BUCKET = 'solr.ucldc'

RE_ARK_FINDER = re.compile('(ark:/\d\d\d\d\d/[^/|\s]*)')
RE_ALPHANUMSPACE = re.compile(r'[^0-9A-Za-z\s]*') #\W include "_" as does A-z

COUCHDOC_TO_SOLR_MAPPING = {
    '_id'       : lambda d: {'harvest_id_s': d['_id']},
    'object'   : lambda d: {'reference_image_md5': d['object']},
    'isShownAt': lambda d: {'url_item': d['isShownAt']},
}

COUCHDOC_SRC_RESOURCE_TO_SOLR_MAPPING = {
    'alternativeTitle'   : lambda d: {'alternative_title': dejson('alternativeTile', d.get('alternativeTitle', None))},
    'contributor' : lambda d: {'contributor': dejson('contributor', d.get('contributor', None))},
    'coverage'    : lambda d: {'coverage': dejson('coverage', d.get('coverage', None))},
    'spatial'     : lambda d: {'coverage': [c['text'] if (isinstance(c, dict)
        and 'text' in c)  else dejson('coverage', c) for c in d['spatial']]},
    'creator'     : lambda d: {'creator': dejson('creator', d.get('creator', None))},
    'date'        : lambda d:  map_date(d),
    'description' : lambda d: {'description': [dejson('description', ds) for ds in d['description']]},
    'extent'      : lambda d: {'extent': dejson('extent', d.get('extent', None))},
    'format'      : lambda d: {'format': dejson('format', d.get('format', None))},
    'genre'       : lambda d: {'genre': dejson('genre', d.get('genre', None))},
    'identifier'  : lambda d: {'identifier': dejson('identifier', d.get('identifier', None))},
    'language'    : lambda d: {'language': [l.get('iso639_3', l.get('name', None)) if isinstance(l, dict) else l for l in d['language']]},
    'publisher'   : lambda d: {'publisher': dejson('publisher', d.get('publisher', None))},
    'relation'    : lambda d: {'relation': dejson('relation', d.get('relation', None))},
    'rights'      : lambda d: {'rights': dejson('rights', d.get('rights', None))},
    'subject'     : lambda d: {'subject': [s['name'] if isinstance(s, dict) else dejson('subject', s) for s in d['subject']]},
    'temporalCoverage'    : lambda d: {'temporal': dejson('temporal', d.get('temporalCoverage', None))},
    'title'       : lambda d: {'title': dejson('title', d.get('title', None))},
    'type'        : lambda d: {'type': dejson('type', d.get('type', None))},
}

COUCHDOC_ORIGINAL_RECORD_TO_SOLR_MAPPING = {
#    'location'        : lambda d: {'location': d.get('location', None)},
    'provenance'      : lambda d: {'provenance': d.get('provenance', None)},
    'dateCopyrighted' : lambda d: {'rights_date': d.get('dateCopyrighted')},
    'rightsHolder'    : lambda d: {'rights_holder': d.get('rightsHolder')},
    'rightsNote'      : lambda d: {'rights_note': d.get('rightsNote')},
    'source'          : lambda d: {'source': d.get('source')},
    'structmap_text'  : lambda d: {'structmap_text': d.get('structmap_text')},
    'structmap_url'   : lambda d: {'structmap_url': d.get('structmap_url')},
    'transcription'   : lambda d: {'transcription': d.get('transcription')},
}

def getjobj(data):
    jobj = None
    try:
        jobj = json.loads(data)
    except ValueError:
        pass
    return jobj

def unpack_if_json(field, data):
    '''If data is a valid json object, attempt to flatten data to a string.
    In general if there is a field 'name' that is the data
    '''
    flatdata = None
    j = getjobj(data)
    if j:
        if 'name' in j:
            flatdata = j['name']
    return flatdata

def dejson(field, data):
    '''de-jsonfy the data.
    For valid json strings, unpack in sensible way?
    '''
    print "{} data type:{}".format(field, type(data))
    dejson_data = None
    if not data:
        return data
    if isinstance(data, list):
        dejson_data = []
        for d in data:
             flatdata = unpack_if_json(field, d)
             if flatdata:
                 dejson_data.append(flatdata)
                 json_str = unicode(json.dumps(j))
            if not isinstance(d, unicode):
                d = unicode(d, 'utf-8')
            print u"FIELD:{} JSON:{} DATA:{}".format(field, json_str, d.encode( 'utf-8'))
    elif isinstance(data, dict):
        print u"DICTIONARY DATA FIELD:{} DATA:{}".format(field, data) 
    else:
        j = getjobj(data)
        json_str = ''
        if j:
            json_str = unicode(json.dumps(j))
        udata = data
        if not isinstance(data, unicode):
            udata = unicode(data, 'utf-8')
        try:
            print u"FIELD:{} JSON:{} DATA:{}".format(field, json_str.encode('utf-8'), udata.encode('utf-8'))
        except UnicodeEncodeError:
            try:
                print u"FIELD:{} JSON:{} DATA:{}".format(field, json_str, data.encode('latin-1'))
            except UnicodeEncodeError:
                print u"FIELD:{} JSON:{} DATA:{}".format(field, json_str, data)
    return data

class UTCtz(datetime.tzinfo):
    def utcoffset(self, dt):
        return datetime.timedelta(hours=0)
    def tzname(self, dt):
        return 'GMT'
    def dst(self, dt):
        return timedelta(0)
UTC = UTCtz()

def make_datetime(dstring):
    '''make a datetime from a valid date string
    Right now formats are YYYY or YYYY-MM-DD
    '''
    dt = None
    if not dstring:
        return dt
    try:
        dint = int(dstring)
        dt = datetime.datetime(dint, 1, 1)
    except ValueError:
        pass
    except TypeError, e:
        print >> sys.stderr, 'Date type err DATA:{} ERROR:{}'.format(dstring, e)
    try:
        strfmt = '%Y-%m-%d'
        dt = datetime.datetime.strptime(dstring, strfmt)
    except ValueError:
        pass
    except TypeError, e:
        print >> sys.stderr, 'Date type err in strptime:{} {}'.format(dstring, e)
    # add UTC as timezone, solrpy looks for tzinfo
    if dt:
        dt = datetime.datetime(dt.year, dt.month, dt.day, tzinfo=UTC)
    return dt

def get_dates_from_date_obj(date_obj):
    '''Return list of display date, start dates, end dates'''
    if isinstance(date_obj, dict):
        date_start = make_datetime(date_obj.get('begin', None))
        date_end = make_datetime(date_obj.get('end', None))
        dates = (date_obj.get('displayDate', None),
                date_start,
                date_end
                )
        return dates
    elif isinstance(date_obj, basestring):
        return date_obj, None, None
    else:
        return None, None, None

def map_date(d):
    date_map = {}
    date_source = d.get('date', None)
    dates = []
    start_date = end_date = None
    dates_start = []
    dates_end = []
    #print >> sys.stderr, 'date_source:{}'.format(date_source)
    if date_source:
        if isinstance(date_source, dict):
            try:
                displayDate, dt_start, dt_end = get_dates_from_date_obj(date_source)
                dates.append(displayDate)
                if dt_start:
                     dates_start.append(dt_start)
                if dt_end:
                    dates_end.append(dt_end)
            except KeyError:
                pass
        else: #should be list
            for dt in date_source:
                displayDate, dt_start, dt_end = get_dates_from_date_obj(dt)
                dates.append(displayDate)
                if dt_start:
                     dates_start.append(dt_start)
                if dt_end:
                    dates_end.append(dt_end)

    date_map['date'] = dates

    dates_start = sorted(dates_start)
    dates_end = sorted(dates_end)
    if len(dates_start):
    	start_date = dates_start[0]
    if len(dates_end):
    	end_date = dates_end[0]
    # fill in start_date == end_date if only one exists
    start_date = end_date if not start_date else start_date
    end_date = start_date if not end_date else end_date
    if start_date:
        #add timezone as UTC, for solrpy (uses astimzone())
        date_map['sort_date_start'] = start_date
        date_map['sort_date_end'] = end_date

    return date_map

def find_ark_in_identifiers(doc):
    identifiers = doc['sourceResource'].get('identifier', None)
    if identifiers:
        for identifier in identifiers:
            match = RE_ARK_FINDER.search(identifier)
            if match:
                return match.group(0)
    return None

def uuid_if_nuxeo(doc):
    collection = doc['originalRecord']['collection'][0]
    harvest_type = collection['harvest_type']
    if harvest_type == 'NUX':
        return doc['originalRecord'].get('uid', None)
    return None

def ucsd_ark(doc):
    #is this UCSD?
    campus = None
    ark = None
    collection = doc['originalRecord']['collection'][0]
{'fraction_non_alphanumeric': 0.14223669923995658, 'fraction_numerical': 0.002746375423133423, 'mean_word_length': 2.757619390448764, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7188450', 'n_tokens_mistral': 5249, 'n_tokens_neox': 5114, 'n_words': 1292}