<commit_before>import datetime
import json
import os
from random import random
from time import sleep
from time import time

import requests
import shortuuid
from sickle import Sickle, oaiexceptions
from sickle.iterator import OAIItemIterator
from sickle.models import ResumptionToken
from sickle.oaiexceptions import NoRecordsMatch
from sickle.response import OAIResponse
from sqlalchemy import or_

import pmh_record
from app import db
from app import logger
from repository import Repository
from util import elapsed
from util import safe_commit


def lookup_endpoint_by_pmh_url(pmh_url_query=None):
    endpoints = Endpoint.query.filter(Endpoint.pmh_url.ilike(u"%{}%".format(pmh_url_query))).all()
    return endpoints


class Endpoint(db.Model):
    id = db.Column(db.Text, primary_key=True)
    id_old = db.Column(db.Text)
    repo_unique_id = db.Column(db.Text, db.ForeignKey(Repository.id))
    pmh_url = db.Column(db.Text)
    pmh_set = db.Column(db.Text)
    last_harvest_started = db.Column(db.DateTime)
    last_harvest_finished = db.Column(db.DateTime)
    most_recent_year_harvested = db.Column(db.DateTime)
    earliest_timestamp = db.Column(db.DateTime)
    email = db.Column(db.Text)  # to help us figure out what kind of repo it is
    error = db.Column(db.Text)
    repo_request_id = db.Column(db.Text)
    harvest_identify_response = db.Column(db.Text)
    harvest_test_recent_dates = db.Column(db.Text)
    sample_pmh_record = db.Column(db.Text)
    contacted = db.Column(db.DateTime)
    contacted_text = db.Column(db.Text)
    policy_promises_no_submitted = db.Column(db.Boolean)
    policy_promises_no_submitted_evidence = db.Column(db.Text)
    ready_to_run = db.Column(db.Boolean)

    meta = db.relationship(
        'Repository',
        lazy='subquery',
        cascade="all",
        backref=db.backref("endpoints", lazy="subquery")
    )

    def __init__(self, **kwargs):
        super(self.__class__, self).__init__(**kwargs)
        if not self.id:
            self.id = shortuuid.uuid()[0:20].lower()

    @property
    def repo(self):
        return self.meta

    def run_diagnostics(self):
        response = test_harvest_url(self.pmh_url)
        self.harvest_identify_response = response["harvest_identify_response"]
        # self.harvest_test_initial_dates = response["harvest_test_initial_dates"]
        self.harvest_test_recent_dates = response["harvest_test_recent_dates"]
        self.sample_pmh_record = response["sample_pmh_record"]

    def harvest(self):
        if not self.harvest_identify_response or not self.harvest_test_recent_dates:
            self.set_identify_and_initial_query()

        today = datetime.datetime.utcnow().date()
        tomorrow = today + datetime.timedelta(days=1)
        yesterday = today - datetime.timedelta(days=1)

        first = (self.most_recent_year_harvested or datetime.datetime(2000, 1, 1)).date()
        first = min(first, yesterday)

        if self.id_old in ['citeseerx.ist.psu.edu/oai2',
                           'europepmc.org/oai.cgi',
                           'www.ncbi.nlm.nih.gov/pmc/oai/oai.cgi',
                           'www.ncbi.nlm.nih.gov/pmc/oai/oai.cgi2']:
            first_plus_delta = first + datetime.timedelta(days=7)
        elif self.id_old in ['export.arxiv.org/oai2']:
            first_plus_delta = first + datetime.timedelta(days=1)
        elif 'osti.gov/oai' in self.pmh_url:
            first_plus_delta = first + datetime.timedelta(days=1)
        else:
            first_plus_delta = first.replace(year=first.year + 1)

        last = min(first_plus_delta, tomorrow)

        # now do the harvesting
        self.call_pmh_endpoint(first=first, last=last)

        # if success, update so we start at next point next time
        if self.error:
            logger.info(u"error so not saving finished info: {}".format(self.error))
        else:
            logger.info(u"success!  saving info")
            self.last_harvest_finished = datetime.datetime.utcnow().isoformat()
            self.most_recent_year_harvested = min(yesterday, last)
            self.last_harvest_started = None

    def get_pmh_record(self, record_id):
        my_sickle = _get_my_sickle(self.pmh_url)
        pmh_input_record = my_sickle.GetRecord(identifier=record_id, metadataPrefix="oai_dc")
        my_pmh_record = pmh_record.PmhRecord()
        my_pmh_record.populate(pmh_input_record)
        my_pmh_record.repo_id = self.id_old  # delete once endpoint_id is populated
        my_pmh_record.endpoint_id = self.id
        return my_pmh_record

    def set_identify_and_initial_query(self):
        if not self.pmh_url:
            self.harvest_identify_response = u"error, no pmh_url given"
            return

        my_sickle = None
        try:
            # set timeout quick... if it can't do this quickly, won't be good for harvesting
            logger.debug(u"getting my_sickle for {}".format(self))
            my_sickle = _get_my_sickle(self.pmh_url, timeout=10)
            my_sickle.Identify()
            self.harvest_identify_response = "SUCCESS!"

        except Exception as e:
            logger.exception(u"in set_identify_and_initial_query")
            self.error = u"error in calling identify: {} {}".format(
                e.__class__.__name__, unicode(e.message).encode("utf-8"))
            if my_sickle:
                self.error += u" calling {}".format(my_sickle.get_http_response_url())

            self.harvest_identify_response = self.error

        last = datetime.datetime.utcnow()
        first = last - datetime.timedelta(days=30)
        self.sample_pmh_record = None
        (pmh_input_record, pmh_records, error) = self.get_pmh_input_record(first, last)
        if error:
            self.harvest_test_recent_dates = error
        elif pmh_input_record:
            self.harvest_test_recent_dates = "SUCCESS!"
            self.sample_pmh_record = json.dumps(pmh_input_record.metadata)
        else:
            self.harvest_test_recent_dates = "error, no pmh_input_records returned"

    def get_pmh_input_record(self, first, last, use_date_default_format=True):
        args = {'metadataPrefix': 'oai_dc'}
        pmh_records = []
        self.error = None

        my_sickle = _get_my_sickle(self.pmh_url)
        logger.info(u"connected to sickle with {}".format(self.pmh_url))

        args['from'] = first.isoformat()[0:10]
        if not use_date_default_format:
            args['from'] += "T00:00:00Z"

        if last:
            args["until"] = last.isoformat()[0:10]
            if not use_date_default_format:
                args['until'] += "T00:00:00Z"

        if self.pmh_set:
            args["set"] = self.pmh_set

        logger.info(u"calling ListRecords with {} {}".format(self.pmh_url, args))<commit_msg>only request one full record in pmh endpoint test<commit_after>import datetime
import json
import os
from random import random
from time import sleep
from time import time

import requests
import shortuuid
from sickle import Sickle, oaiexceptions
from sickle.iterator import OAIItemIterator
from sickle.models import ResumptionToken
from sickle.oaiexceptions import NoRecordsMatch
from sickle.response import OAIResponse
from sqlalchemy import or_

import pmh_record
from app import db
from app import logger
from repository import Repository
from util import elapsed
from util import safe_commit


def lookup_endpoint_by_pmh_url(pmh_url_query=None):
    endpoints = Endpoint.query.filter(Endpoint.pmh_url.ilike(u"%{}%".format(pmh_url_query))).all()
    return endpoints


class Endpoint(db.Model):
    id = db.Column(db.Text, primary_key=True)
    id_old = db.Column(db.Text)
    repo_unique_id = db.Column(db.Text, db.ForeignKey(Repository.id))
    pmh_url = db.Column(db.Text)
    pmh_set = db.Column(db.Text)
    last_harvest_started = db.Column(db.DateTime)
    last_harvest_finished = db.Column(db.DateTime)
    most_recent_year_harvested = db.Column(db.DateTime)
    earliest_timestamp = db.Column(db.DateTime)
    email = db.Column(db.Text)  # to help us figure out what kind of repo it is
    error = db.Column(db.Text)
    repo_request_id = db.Column(db.Text)
    harvest_identify_response = db.Column(db.Text)
    harvest_test_recent_dates = db.Column(db.Text)
    sample_pmh_record = db.Column(db.Text)
    contacted = db.Column(db.DateTime)
    contacted_text = db.Column(db.Text)
    policy_promises_no_submitted = db.Column(db.Boolean)
    policy_promises_no_submitted_evidence = db.Column(db.Text)
    ready_to_run = db.Column(db.Boolean)

    meta = db.relationship(
        'Repository',
        lazy='subquery',
        cascade="all",
        backref=db.backref("endpoints", lazy="subquery")
    )

    def __init__(self, **kwargs):
        super(self.__class__, self).__init__(**kwargs)
        if not self.id:
            self.id = shortuuid.uuid()[0:20].lower()

    @property
    def repo(self):
        return self.meta

    def run_diagnostics(self):
        response = test_harvest_url(self.pmh_url)
        self.harvest_identify_response = response["harvest_identify_response"]
        # self.harvest_test_initial_dates = response["harvest_test_initial_dates"]
        self.harvest_test_recent_dates = response["harvest_test_recent_dates"]
        self.sample_pmh_record = response["sample_pmh_record"]

    def harvest(self):
        if not self.harvest_identify_response or not self.harvest_test_recent_dates:
            self.set_identify_and_initial_query()

        today = datetime.datetime.utcnow().date()
        tomorrow = today + datetime.timedelta(days=1)
        yesterday = today - datetime.timedelta(days=1)

        first = (self.most_recent_year_harvested or datetime.datetime(2000, 1, 1)).date()
        first = min(first, yesterday)

        if self.id_old in ['citeseerx.ist.psu.edu/oai2',
                           'europepmc.org/oai.cgi',
                           'www.ncbi.nlm.nih.gov/pmc/oai/oai.cgi',
                           'www.ncbi.nlm.nih.gov/pmc/oai/oai.cgi2']:
            first_plus_delta = first + datetime.timedelta(days=7)
        elif self.id_old in ['export.arxiv.org/oai2']:
            first_plus_delta = first + datetime.timedelta(days=1)
        elif 'osti.gov/oai' in self.pmh_url:
            first_plus_delta = first + datetime.timedelta(days=1)
        else:
            first_plus_delta = first.replace(year=first.year + 1)

        last = min(first_plus_delta, tomorrow)

        # now do the harvesting
        self.call_pmh_endpoint(first=first, last=last)

        # if success, update so we start at next point next time
        if self.error:
            logger.info(u"error so not saving finished info: {}".format(self.error))
        else:
            logger.info(u"success!  saving info")
            self.last_harvest_finished = datetime.datetime.utcnow().isoformat()
            self.most_recent_year_harvested = min(yesterday, last)
            self.last_harvest_started = None

    def get_pmh_record(self, record_id):
        my_sickle = _get_my_sickle(self.pmh_url)
        pmh_input_record = my_sickle.GetRecord(identifier=record_id, metadataPrefix="oai_dc")
        my_pmh_record = pmh_record.PmhRecord()
        my_pmh_record.populate(pmh_input_record)
        my_pmh_record.repo_id = self.id_old  # delete once endpoint_id is populated
        my_pmh_record.endpoint_id = self.id
        return my_pmh_record

    def set_identify_and_initial_query(self):
        if not self.pmh_url:
            self.harvest_identify_response = u"error, no pmh_url given"
            return

        my_sickle = None
        try:
            # set timeout quick... if it can't do this quickly, won't be good for harvesting
            logger.debug(u"getting my_sickle for {}".format(self))
            my_sickle = _get_my_sickle(self.pmh_url, timeout=10)
            my_sickle.Identify()
            self.harvest_identify_response = "SUCCESS!"

        except Exception as e:
            logger.exception(u"in set_identify_and_initial_query")
            self.error = u"error in calling identify: {} {}".format(
                e.__class__.__name__, unicode(e.message).encode("utf-8"))
            if my_sickle:
                self.error += u" calling {}".format(my_sickle.get_http_response_url())

            self.harvest_identify_response = self.error

        self.sample_pmh_record = None

        try:
            sample_pmh_record = self.get_recent_pmh_record()
            if sample_pmh_record:
                self.harvest_test_recent_dates = "SUCCESS!"
                self.sample_pmh_record = json.dumps(sample_pmh_record.metadata)
            else:
                self.harvest_test_recent_dates = "error, no pmh_input_records returned"
        except Exception as e:
            self.error = u"error in get_recent_pmh_record: {} {}".format(
                e.__class__.__name__, unicode(e.message).encode("utf-8"))
            self.harvest_test_recent_dates = self.error

    def get_recent_pmh_record(self):
        last = datetime.datetime.utcnow()
        first = last - datetime.timedelta(days=30)

        args = {'metadataPrefix': 'oai_dc'}

        my_sickle = _get_my_sickle(self.pmh_url)
        logger.info(u"connected to sickle with {}".format(self.pmh_url))

        args['from'] = first.isoformat()[0:10]
        args["until"] = last.isoformat()[0:10]

        if self.pmh_set:
            args["set"] = self.pmh_set

        logger.info(u"calling ListIdentifiers with {} {}".format(self.pmh_url, args))
        try:
            pmh_identifiers = my_sickle.ListIdentifiers(ignore_deleted=True, **args)
            pmh_identifier = self.safe_get_next_record(pmh_identifiers)
            if pmh_identifier:
                return my_sickle.GetRecord(identifier=pmh_identifier.identifier, metadataPrefix='oai_dc')
            else:
                return None
        except NoRecordsMatch:
            logger.info(u"no records with {} {}".format(self.pmh_url, args))
            return None

    def get_pmh_input_record(self, first, last, use_date_default_format=True):
        args = {'metadataPrefix': 'oai_dc'}
        pmh_records = []
        self.error = None

        my_sickle = _get_my_sickle(self.pmh_url)
        logger.info(u"connected to sickle with {}".format(self.pmh_url))

        args['from'] = first.isoformat()[0:10]
        if not use_date_default_format:
            args['from'] += "T00:00:00Z"

        if last:
            args["until"] = last.isoformat()[0:10]
            if not use_date_default_format:
                args['until'] += "T00:00:00Z"

        if self.pmh_set:
            args["set"] = self.pmh_set

        logger.info(u"calling ListRecords with {} {}".format(self.pmh_url, args))
{'fraction_non_alphanumeric': 0.10057764186204553, 'fraction_numerical': 0.006048250084947333, 'mean_word_length': 2.761758691206544, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6381412', 'n_tokens_mistral': 4861, 'n_tokens_neox': 4694, 'n_words': 884}