<commit_before># -*- coding: utf8 -*-
#

import re

import shortuuid
from enum import Enum

from app import db
from app import logger
from pdf_url import PdfUrl
from reported_noncompliant_copies import is_reported_noncompliant_url
from util import clean_doi
from util import is_doi_url


def url_sort_score(url):
    url_lower = url.lower()

    score = 0

    # pmc results are better than IR results, if we've got them
    if "europepmc.org" in url_lower:
        score += -50

    if "/pmc/" in url_lower:
        score += -45

    # arxiv results are better than IR results, if we've got them
    if "arxiv" in url_lower:
        score += -40

    if ".edu" in url_lower:
        score += -30

    if "citeseerx" in url_lower:
        score += +10

    # ftp is really bad
    if "ftp" in url_lower:
        score += +60

    # break ties
    if "pdf" in url_lower:
        score += -1

    # otherwise whatever we've got
    return score


def validate_pdf_urls(open_locations):
    unvalidated = [x for x in open_locations if x.pdf_url_valid is None]

    if unvalidated:
        bad_pdf_urls = {
            x.url for x in
            PdfUrl.query.filter(
                PdfUrl.url.in_([x.pdf_url for x in unvalidated]),
                PdfUrl.is_pdf.is_(False)
            ).all()
        }

        for location in unvalidated:
            location.pdf_url_valid = (
                location.pdf_url not in bad_pdf_urls
                # get rid of this, make PDF checker more robust
                or 'journal.csj.jp/doi/pdf' in location.pdf_url
            )

            if not location.pdf_url_valid:
                logger.info(u'excluding location with bad pdf url: {}'.format(location))


class OAStatus(Enum):
    closed = 'closed'
    green = 'green'
    bronze = 'bronze'
    hybrid = 'hybrid'
    gold = 'gold'


def oa_status_sort_key(location):
    keys = {
        OAStatus.closed:    0,
        OAStatus.green:     1,
        OAStatus.bronze:    2,
        OAStatus.hybrid:    3,
        OAStatus.gold:      4,
    }

    return keys.get(location.oa_status, 0)



class OpenLocation(db.Model):
    id = db.Column(db.Text, primary_key=True)
    pub_id = db.Column(db.Text, db.ForeignKey('pub.id'))
    doi = db.Column(db.Text)  # denormalized from Publication for ease of interpreting

    pdf_url = db.Column(db.Text)
    metadata_url = db.Column(db.Text)
    license = db.Column(db.Text)
    evidence = db.Column(db.Text)
    updated = db.Column(db.DateTime)
    error = db.Column(db.Text)

    def __init__(self, **kwargs):
        self.id = shortuuid.uuid()[0:20]
        self.doi = ""
        self.match = {}
        self.pmh_id = None
        self.endpoint_id = None
        self.base_doc = None
        self.version = None
        self.error = ""
        self.pdf_url_valid = None
        self.institution = None
        super(OpenLocation, self).__init__(**kwargs)

    @property
    def has_license(self):
        if not self.license:
            return False
        if self.license == "unknown":
            return False
        return True

    @property
    def best_url(self):
        if self.pdf_url:
            return self.pdf_url
        return self.metadata_url

    @property
    def best_url_is_pdf(self):
        if not self.best_url:
            return None
        if self.pdf_url:
            return True
        return False

    @property
    def is_reported_noncompliant(self):
        if is_reported_noncompliant_url(self.doi, self.pdf_url) or is_reported_noncompliant_url(self.doi, self.metadata_url):
            return True
        return False

    @property
    def is_gold(self):
        return self.display_evidence.startswith('oa journal')

    @property
    def is_green(self):
        return self.display_evidence.startswith('oa repository')

    @property
    def is_hybrid(self):
        return self.display_evidence.startswith("open") and self.display_evidence != 'open (via free pdf)'

    @property
    def is_bronze(self):
        if self.display_evidence == 'open (via free pdf)':
            return True

        if is_doi_url(self.best_url):
            return (
                clean_doi(self.best_url, return_none_if_error=True) == self.doi
                and not (self.is_gold or self.is_hybrid)
            )

        return False

    @property
    def display_evidence(self):
        return self.evidence or ''

    @property
    def host_type_calculated(self):
        if self.is_gold or self.is_hybrid or self.is_bronze:
            return "publisher"
        return "repository"

    @property
    def host_type(self):
        if hasattr(self, "host_type_set"):
            return self.host_type_set<commit_msg>base hybrid/bronze distinction on license explicitly<commit_after># -*- coding: utf8 -*-
#

import re

import shortuuid
from enum import Enum

from app import db
from app import logger
from pdf_url import PdfUrl
from reported_noncompliant_copies import is_reported_noncompliant_url
from util import clean_doi
from util import is_doi_url


def url_sort_score(url):
    url_lower = url.lower()

    score = 0

    # pmc results are better than IR results, if we've got them
    if "europepmc.org" in url_lower:
        score += -50

    if "/pmc/" in url_lower:
        score += -45

    # arxiv results are better than IR results, if we've got them
    if "arxiv" in url_lower:
        score += -40

    if ".edu" in url_lower:
        score += -30

    if "citeseerx" in url_lower:
        score += +10

    # ftp is really bad
    if "ftp" in url_lower:
        score += +60

    # break ties
    if "pdf" in url_lower:
        score += -1

    # otherwise whatever we've got
    return score


def validate_pdf_urls(open_locations):
    unvalidated = [x for x in open_locations if x.pdf_url_valid is None]

    if unvalidated:
        bad_pdf_urls = {
            x.url for x in
            PdfUrl.query.filter(
                PdfUrl.url.in_([x.pdf_url for x in unvalidated]),
                PdfUrl.is_pdf.is_(False)
            ).all()
        }

        for location in unvalidated:
            location.pdf_url_valid = (
                location.pdf_url not in bad_pdf_urls
                # get rid of this, make PDF checker more robust
                or 'journal.csj.jp/doi/pdf' in location.pdf_url
            )

            if not location.pdf_url_valid:
                logger.info(u'excluding location with bad pdf url: {}'.format(location))


class OAStatus(Enum):
    closed = 'closed'
    green = 'green'
    bronze = 'bronze'
    hybrid = 'hybrid'
    gold = 'gold'


def oa_status_sort_key(location):
    keys = {
        OAStatus.closed:    0,
        OAStatus.green:     1,
        OAStatus.bronze:    2,
        OAStatus.hybrid:    3,
        OAStatus.gold:      4,
    }

    return keys.get(location.oa_status, 0)



class OpenLocation(db.Model):
    id = db.Column(db.Text, primary_key=True)
    pub_id = db.Column(db.Text, db.ForeignKey('pub.id'))
    doi = db.Column(db.Text)  # denormalized from Publication for ease of interpreting

    pdf_url = db.Column(db.Text)
    metadata_url = db.Column(db.Text)
    license = db.Column(db.Text)
    evidence = db.Column(db.Text)
    updated = db.Column(db.DateTime)
    error = db.Column(db.Text)

    def __init__(self, **kwargs):
        self.id = shortuuid.uuid()[0:20]
        self.doi = ""
        self.match = {}
        self.pmh_id = None
        self.endpoint_id = None
        self.base_doc = None
        self.version = None
        self.error = ""
        self.pdf_url_valid = None
        self.institution = None
        super(OpenLocation, self).__init__(**kwargs)

    @property
    def has_license(self):
        if not self.license:
            return False
        if self.license == "unknown":
            return False
        return True

    @property
    def best_url(self):
        if self.pdf_url:
            return self.pdf_url
        return self.metadata_url

    @property
    def best_url_is_pdf(self):
        if not self.best_url:
            return None
        if self.pdf_url:
            return True
        return False

    @property
    def is_reported_noncompliant(self):
        if is_reported_noncompliant_url(self.doi, self.pdf_url) or is_reported_noncompliant_url(self.doi, self.metadata_url):
            return True
        return False

    @property
    def is_gold(self):
        return self.display_evidence.startswith('oa journal')

    @property
    def is_green(self):
        return self.display_evidence.startswith('oa repository')

    @property
    def is_hybrid(self):
        return self.display_evidence.startswith("open") and self.has_license

    @property
    def is_bronze(self):
        if self.display_evidence.startswith("open") and not self.has_license:
            return True

        if is_doi_url(self.best_url):
            return (
                clean_doi(self.best_url, return_none_if_error=True) == self.doi
                and not (self.is_gold or self.is_hybrid)
            )

        return False

    @property
    def display_evidence(self):
        return self.evidence or ''

    @property
    def host_type_calculated(self):
        if self.is_gold or self.is_hybrid or self.is_bronze:
            return "publisher"
        return "repository"

    @property
    def host_type(self):
        if hasattr(self, "host_type_set"):
            return self.host_type_set
{'fraction_non_alphanumeric': 0.08952846219201359, 'fraction_numerical': 0.005097706032285472, 'mean_word_length': 2.06742671009772, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2799059', 'n_tokens_mistral': 3087, 'n_tokens_neox': 2882, 'n_words': 759}