<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import datetime
import random
import re

import shortuuid
from sqlalchemy import sql
from sqlalchemy.dialects.postgresql import JSONB

from app import db
from app import logger
from http_cache import http_get
from oa_local import find_normalized_license
from pdf_to_text import convert_pdf_to_txt
from oa_pmc import query_pmc
import oa_page
from util import is_pmc
from webpage import PmhRepoWebpage, PublisherWebpage


DEBUG_BASE = False


class PmhVersionFirstAvailable(db.Model):
    endpoint_id = db.Column(db.Text, primary_key=True)
    pmh_id = db.Column(db.Text, primary_key=True)
    scrape_version = db.Column(db.Text, primary_key=True)
    first_available = db.Column(db.DateTime)
    url = db.Column(db.Text)


class PageNew(db.Model):
    id = db.Column(db.Text, primary_key=True)
    url = db.Column(db.Text)
    pmh_id = db.Column(db.Text, db.ForeignKey("pmh_record.id"))
    repo_id = db.Column(db.Text)  # delete once endpoint_id is populated
    endpoint_id = db.Column(db.Text, db.ForeignKey("endpoint.id"))
    doi = db.Column(db.Text, db.ForeignKey("pub.id"))
    title = db.Column(db.Text)
    normalized_title = db.Column(db.Text, db.ForeignKey("pub.normalized_title"))
    authors = db.Column(JSONB)
    record_timestamp = db.Column(db.DateTime)

    scrape_updated = db.Column(db.DateTime)
    scrape_pdf_url = db.Column(db.Text)
    scrape_metadata_url = db.Column(db.Text)
    scrape_version = db.Column(db.Text)
    scrape_license = db.Column(db.Text)
    num_pub_matches = db.Column(db.Numeric)

    error = db.Column(db.Text)
    updated = db.Column(db.DateTime)

    rand = db.Column(db.Numeric)

    match_type = db.Column(db.Text)

    endpoint = db.relationship(
        'Endpoint',
        lazy='subquery',
        uselist=None,
        cascade="",
        viewonly=True
    )

    pmh_record = db.relationship(
        'PmhRecord',
        lazy='subquery',
        uselist=None,
        cascade="",
        viewonly=True
    )

    __mapper_args__ = {
        "polymorphic_on": match_type,
        "polymorphic_identity": "page_new"
    }

    def __init__(self, **kwargs):
        self.id = shortuuid.uuid()[0:20]
        self.error = ""
        self.rand = random.random()
        self.updated = datetime.datetime.utcnow().isoformat()
        super(PageNew, self).__init__(**kwargs)

    @property
    def first_available(self):
        if self.pmh_record:
            lookup = PmhVersionFirstAvailable.query.filter(
                PmhVersionFirstAvailable.pmh_id == self.pmh_record.bare_pmh_id,
                PmhVersionFirstAvailable.endpoint_id == self.pmh_record.endpoint_id,
                PmhVersionFirstAvailable.scrape_version == self.scrape_version
            ).first()

            if lookup:
                return lookup.first_available.date()

        if self.pmcid:
            return self.pmc_first_available_date()

        if self.pmh_record:
            return self.pmh_record.record_timestamp.date()

        return None

    @property
    def is_open(self):
        return self.scrape_metadata_url or self.scrape_pdf_url

    @property
    def is_pmc(self):
        return self.url and is_pmc(self.url)

    @property
    def pmcid(self):
        if not self.is_pmc:
            return None
        matches = re.findall(u"(pmc\d+)", self.url, re.IGNORECASE)
        if not matches:
            return None
        return matches[0].lower()

    def get_pmh_record_url(self):
        return self.endpoint and self.pmh_record and u"{}?verb=GetRecord&metadataPrefix={}&identifier={}".format(<commit_msg>remove europe pmc api call in update function<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import datetime
import random
import re

import shortuuid
from sqlalchemy import sql
from sqlalchemy.dialects.postgresql import JSONB

from app import db
from app import logger
from http_cache import http_get
from oa_local import find_normalized_license
from pdf_to_text import convert_pdf_to_txt
from oa_pmc import query_pmc
import oa_page
from util import is_pmc
from webpage import PmhRepoWebpage, PublisherWebpage


DEBUG_BASE = False


class PmhVersionFirstAvailable(db.Model):
    endpoint_id = db.Column(db.Text, primary_key=True)
    pmh_id = db.Column(db.Text, primary_key=True)
    scrape_version = db.Column(db.Text, primary_key=True)
    first_available = db.Column(db.DateTime)
    url = db.Column(db.Text)


class PageNew(db.Model):
    id = db.Column(db.Text, primary_key=True)
    url = db.Column(db.Text)
    pmh_id = db.Column(db.Text, db.ForeignKey("pmh_record.id"))
    repo_id = db.Column(db.Text)  # delete once endpoint_id is populated
    endpoint_id = db.Column(db.Text, db.ForeignKey("endpoint.id"))
    doi = db.Column(db.Text, db.ForeignKey("pub.id"))
    title = db.Column(db.Text)
    normalized_title = db.Column(db.Text, db.ForeignKey("pub.normalized_title"))
    authors = db.Column(JSONB)
    record_timestamp = db.Column(db.DateTime)

    scrape_updated = db.Column(db.DateTime)
    scrape_pdf_url = db.Column(db.Text)
    scrape_metadata_url = db.Column(db.Text)
    scrape_version = db.Column(db.Text)
    scrape_license = db.Column(db.Text)
    num_pub_matches = db.Column(db.Numeric)

    error = db.Column(db.Text)
    updated = db.Column(db.DateTime)

    rand = db.Column(db.Numeric)

    match_type = db.Column(db.Text)

    endpoint = db.relationship(
        'Endpoint',
        lazy='subquery',
        uselist=None,
        cascade="",
        viewonly=True
    )

    pmh_record = db.relationship(
        'PmhRecord',
        lazy='subquery',
        uselist=None,
        cascade="",
        viewonly=True
    )

    __mapper_args__ = {
        "polymorphic_on": match_type,
        "polymorphic_identity": "page_new"
    }

    def __init__(self, **kwargs):
        self.id = shortuuid.uuid()[0:20]
        self.error = ""
        self.rand = random.random()
        self.updated = datetime.datetime.utcnow().isoformat()
        super(PageNew, self).__init__(**kwargs)

    @property
    def first_available(self):
        if self.pmh_record:
            lookup = PmhVersionFirstAvailable.query.filter(
                PmhVersionFirstAvailable.pmh_id == self.pmh_record.bare_pmh_id,
                PmhVersionFirstAvailable.endpoint_id == self.pmh_record.endpoint_id,
                PmhVersionFirstAvailable.scrape_version == self.scrape_version
            ).first()

            if lookup:
                return lookup.first_available.date()

        # if self.pmcid:
        #     return self.pmc_first_available_date()

        if self.pmh_record:
            return self.pmh_record.record_timestamp.date()

        return None

    @property
    def is_open(self):
        return self.scrape_metadata_url or self.scrape_pdf_url

    @property
    def is_pmc(self):
        return self.url and is_pmc(self.url)

    @property
    def pmcid(self):
        if not self.is_pmc:
            return None
        matches = re.findall(u"(pmc\d+)", self.url, re.IGNORECASE)
        if not matches:
            return None
        return matches[0].lower()

    def get_pmh_record_url(self):
        return self.endpoint and self.pmh_record and u"{}?verb=GetRecord&metadataPrefix={}&identifier={}".format(
{'fraction_non_alphanumeric': 0.10640637888369535, 'fraction_numerical': 0.0013747594171020072, 'mean_word_length': 2.824921135646688, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4747197', 'n_tokens_mistral': 2456, 'n_tokens_neox': 2360, 'n_words': 452}