<commit_before>from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import deferred
from sqlalchemy import or_
from sqlalchemy import sql
from sqlalchemy import text
from sqlalchemy import orm
from executor import execute
import requests
from time import sleep
from time import time
import datetime
import shortuuid
from urllib import quote
import os
import re
import geoip2.webservice

from app import logger
from app import db
from util import elapsed
from util import safe_commit
from util import clean_doi
from publication import CrossrefApi
from put_pmh_in_db import pmh_to_db

# CREATE TABLE doi_queue_dates as (select s as id, random() as rand, false as enqueued, null::timestamp as finished, null::timestamp as started, null::text as dyno FROM generate_series
#         ( '1980-01-01'::timestamp
#         , '2017-08-22'::timestamp
#         , '1 day'::interval) s);

# truncate doi_queue_dates
# insert into doi_queue_dates (select s as id, random() as rand, false as enqueued, null::timestamp as finished, null::timestamp as started, null::text as dyno FROM generate_series
#         ( '1950-01-01'::timestamp
#         , '2017-10-08'::timestamp
#         , '1 day'::interval) s);


class DateRange(db.Model):
    id = db.Column(db.DateTime, primary_key=True)
    # end_date = db.Column(db.DateTime)

    @property
    def first(self):
        return self.id

    @property
    def first_day(self):
        return self.id.isoformat()[0:10]

    @property
    def last_day(self):
        return self.last.isoformat()[0:10]

    @property
    def last(self):
        return self.first + datetime.timedelta(days=1)

    def get_pmh_events(self, rows=100):
        urls = [
            "http://cds.cern.ch/oai2d",
            "https://api.archives-ouvertes.fr/oai/hal",
            # "http://oai.repec.openlib.org/",
            "https://oai.datacite.org/oai",
            "http://deepblue.lib.umich.edu/dspace-oai/request",
            "https://depot.erudit.org/dspace-oai/request",
            "http://www.econstor.eu/dspace-oai/request",
            "http://oai.openedition.org",
            "http://digital.csic.es/dspace-oai/request",
            "http://texashistory.unt.edu/oai/",
            # "http://oai.dtic.mil/oai/oai",
            "http://www.intechopen.com/oai/?",
            "http://quod.lib.umich.edu/cgi/o/oai/oai",
            "http://eprints.whiterose.ac.uk/cgi/oai2",
            "http://ora.ox.ac.uk/oai",
            "http://www.acervodigital.unesp.br/oai/request",
            "http://authors.library.caltech.edu/cgi/oai2",
            "http://pubman.mpdl.mpg.de/escidoc-oaiprovider/",
            "http://dash.harvard.edu/oai/request",
            "http://ir.lib.ntust.edu.tw/dspace-oai/request",
            "https://biblio.ugent.be/oai",
            "https://spiral.imperial.ac.uk/dspace-oai/request",
            "http://orbit.dtu.dk/ws/oai",
            "http://ntrs.nasa.gov/oai",
            "https://www.repository.cam.ac.uk/oai/request",
            "http://infoscience.epfl.ch/oai2d/",
            "http://digitalcommons.unl.edu/do/oai/",
            "http://www.zora.uzh.ch/cgi/oai2",
            "http://dspace.mit.edu/oai/request"
            # "http://export.arxiv.org/oai2",
            # "https://www.ncbi.nlm.nih.gov/pmc/oai/oai.cgi",
            # "http://citeseerx.ist.psu.edu/oai2"
        ]

        for url in urls:
            logger.info(u"starting with url: {}, first={}, last={}".format(url, self.first_day, self.last_day))
            pmh_to_db(url=url, first=self.first_day, last=self.last_day)<commit_msg>new urls<commit_after>from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import deferred
from sqlalchemy import or_
from sqlalchemy import sql
from sqlalchemy import text
from sqlalchemy import orm
from executor import execute
import requests
from time import sleep
from time import time
import datetime
import shortuuid
from urllib import quote
import os
import re
import geoip2.webservice

from app import logger
from app import db
from util import elapsed
from util import safe_commit
from util import clean_doi
from publication import CrossrefApi
from put_pmh_in_db import pmh_to_db

# CREATE TABLE doi_queue_dates as (select s as id, random() as rand, false as enqueued, null::timestamp as finished, null::timestamp as started, null::text as dyno FROM generate_series
#         ( '1980-01-01'::timestamp
#         , '2017-08-22'::timestamp
#         , '1 day'::interval) s);

# truncate doi_queue_dates
# insert into doi_queue_dates (select s as id, random() as rand, false as enqueued, null::timestamp as finished, null::timestamp as started, null::text as dyno FROM generate_series
#         ( '1950-01-01'::timestamp
#         , '2017-10-08'::timestamp
#         , '1 day'::interval) s);


class DateRange(db.Model):
    id = db.Column(db.DateTime, primary_key=True)
    # end_date = db.Column(db.DateTime)

    @property
    def first(self):
        return self.id

    @property
    def first_day(self):
        return self.id.isoformat()[0:10]

    @property
    def last_day(self):
        return self.last.isoformat()[0:10]

    @property
    def last(self):
        return self.first + datetime.timedelta(days=1)

    def get_pmh_events(self, rows=100):
        urls = [
        "http://www.lareferencia.info:8080/oai/provider",
        "http://oai.bnf.fr/oai2/OAIHandler",
        "http://digitallibrary.usc.edu/oai/oai.php",
        "https://kuscholarworks.ku.edu/dspace-oai/request",
        "http://discovery.ucl.ac.uk/cgi/oai2",
        "http://www.diva-portal.org/dice/oai",
        "http://dlisv03.media.osaka-cu.ac.jp/il/oai_repository/repository",
        "http://dspace.library.uu.nl/oai/request",
        "http://oai.persee.fr/c/ext/prescript/oai"

            # "http://cds.cern.ch/oai2d",
            # "https://api.archives-ouvertes.fr/oai/hal",
            # "https://oai.datacite.org/oai",
            # "http://deepblue.lib.umich.edu/dspace-oai/request",
            # "https://depot.erudit.org/dspace-oai/request",
            # "http://www.econstor.eu/dspace-oai/request",
            # "http://oai.openedition.org",
            # "http://digital.csic.es/dspace-oai/request",
            # "http://texashistory.unt.edu/oai/",
            # "http://www.intechopen.com/oai/?",
            # "http://quod.lib.umich.edu/cgi/o/oai/oai",
            # "http://eprints.whiterose.ac.uk/cgi/oai2",
            # "http://ora.ox.ac.uk/oai",
            # "http://www.acervodigital.unesp.br/oai/request",
            # "http://authors.library.caltech.edu/cgi/oai2",
            # "http://pubman.mpdl.mpg.de/escidoc-oaiprovider/",
            # "http://dash.harvard.edu/oai/request",
            # "http://ir.lib.ntust.edu.tw/dspace-oai/request",
            # "https://biblio.ugent.be/oai",
            # "https://spiral.imperial.ac.uk/dspace-oai/request",
            # "http://orbit.dtu.dk/ws/oai",
            # "http://ntrs.nasa.gov/oai",
            # "https://www.repository.cam.ac.uk/oai/request",
            # "http://infoscience.epfl.ch/oai2d/",
            # "http://digitalcommons.unl.edu/do/oai/",
            # "http://www.zora.uzh.ch/cgi/oai2",
            # "http://dspace.mit.edu/oai/request"
            # "http://export.arxiv.org/oai2",
            # "https://www.ncbi.nlm.nih.gov/pmc/oai/oai.cgi",
            # "http://citeseerx.ist.psu.edu/oai2"

            # "http://oai.repec.openlib.org/",
            # "http://oai.dtic.mil/oai/oai",

        ]

        for url in urls:
            logger.info(u"starting with url: {}, first={}, last={}".format(url, self.first_day, self.last_day))
            pmh_to_db(url=url, first=self.first_day, last=self.last_day)
{'fraction_non_alphanumeric': 0.15346663204362504, 'fraction_numerical': 0.014541677486367177, 'mean_word_length': 2.366695804195804, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 15, 'lorem ipsum': 0, 'www.': 14, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '860090', 'n_tokens_mistral': 2824, 'n_tokens_neox': 2676, 'n_words': 447}