<commit_before>import datetime
import json
import random
import re
from collections import Counter
from collections import defaultdict
from dateutil.relativedelta import relativedelta
from threading import Thread
from time import time

import requests
from enum import Enum
from lxml import etree
from sqlalchemy import orm, sql
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm.attributes import flag_modified

import endpoint
import pmh_record
import oa_evidence
import oa_local
import oa_manual
import oa_page
import page
from abstract import Abstract
from app import db
from app import logger
from http_cache import get_session_id
from journal import Journal
from oa_pmc import query_pmc
from open_location import OpenLocation, validate_pdf_urls, OAStatus, oa_status_sort_key
from pdf_url import PdfUrl
from pmh_record import PmhRecord
from pmh_record import title_is_too_common
from pmh_record import title_is_too_short
from reported_noncompliant_copies import reported_noncompliant_url_fragments
from util import NoDoiException
from util import clean_doi, is_pmc, clamp
from util import elapsed
from util import normalize
from util import normalize_title
from util import safe_commit
from webpage import PublisherWebpage


def build_new_pub(doi, crossref_api):
    my_pub = Pub(id=doi, crossref_api_raw_new=crossref_api)
    my_pub.title = my_pub.crossref_title
    my_pub.normalized_title = normalize_title(my_pub.title)
    return my_pub


def add_new_pubs(pubs_to_commit):
    if not pubs_to_commit:
        return []

    pubs_indexed_by_id = dict((my_pub.id, my_pub) for my_pub in pubs_to_commit)
    ids_already_in_db = [
        id_tuple[0] for id_tuple in db.session.query(Pub.id).filter(Pub.id.in_(pubs_indexed_by_id.keys())).all()
    ]
    pubs_to_add_to_db = []

    for (pub_id, my_pub) in pubs_indexed_by_id.iteritems():
        if pub_id in ids_already_in_db:
            # merge if we need to
            pass
        else:
            pubs_to_add_to_db.append(my_pub)
            # logger.info(u"adding new pub {}".format(my_pub.id))

    if pubs_to_add_to_db:
        logger.info(u"adding {} pubs".format(len(pubs_to_add_to_db)))
        db.session.add_all(pubs_to_add_to_db)
        safe_commit(db)
    return pubs_to_add_to_db


def call_targets_in_parallel(targets):
    if not targets:
        return

    # logger.info(u"calling", targets)
    threads = []
    for target in targets:
        process = Thread(target=target, args=[])
        process.start()
        threads.append(process)
    for process in threads:
        try:
            process.join(timeout=60*10)
        except (KeyboardInterrupt, SystemExit):
            pass
        except Exception as e:
            logger.exception(u"thread Exception {} in call_targets_in_parallel. continuing.".format(e))
    # logger.info(u"finished the calls to {}".format(targets))


def call_args_in_parallel(target, args_list):
    # logger.info(u"calling", targets)
    threads = []
    for args in args_list:
        process = Thread(target=target, args=args)
        process.start()
        threads.append(process)
    for process in threads:
        try:
            process.join(timeout=60*10)
        except (KeyboardInterrupt, SystemExit):
            pass
        except Exception as e:
            logger.exception(u"thread Exception {} in call_args_in_parallel. continuing.".format(e))
    # logger.info(u"finished the calls to {}".format(targets))


def lookup_product_by_doi(doi):
    biblio = {"doi": doi}
    return lookup_product(**biblio)


def lookup_product(**biblio):
    my_pub = None
    if "doi" in biblio and biblio["doi"]:
        doi = clean_doi(biblio["doi"])
        my_pub = Pub.query.get(doi)
        if my_pub:
            # logger.info(u"found {} in pub db table!".format(my_pub.id))
            my_pub.reset_vars()
        else:
            raise NoDoiException
        #     my_pub = Crossref(**biblio)
        #     logger.info(u"didn't find {} in crossref db table".format(my_pub))

    return my_pub


def refresh_pub(my_pub, do_commit=False):
    my_pub.run_with_hybrid()
    db.session.merge(my_pub)
    if do_commit:
        safe_commit(db)
    return my_pub


def thread_result_wrapper(func, args, res):
    res.append(func(*args))


# get rid of this when we get rid of POST endpoint
# for now, simplify it so it just calls the single endpoint<commit_msg>map fake JSTOR DOIs to real ones<commit_after>import datetime
import json
import random
import re
from collections import Counter
from collections import defaultdict
from dateutil.relativedelta import relativedelta
from threading import Thread
from time import time

import requests
from enum import Enum
from lxml import etree
from sqlalchemy import orm, sql
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm.attributes import flag_modified

import endpoint
import pmh_record
import oa_evidence
import oa_local
import oa_manual
import oa_page
import page
from abstract import Abstract
from app import db
from app import logger
from http_cache import get_session_id
from journal import Journal
from oa_pmc import query_pmc
from open_location import OpenLocation, validate_pdf_urls, OAStatus, oa_status_sort_key
from pdf_url import PdfUrl
from pmh_record import PmhRecord
from pmh_record import title_is_too_common
from pmh_record import title_is_too_short
from reported_noncompliant_copies import reported_noncompliant_url_fragments
from util import NoDoiException
from util import clean_doi, is_pmc, clamp
from util import elapsed
from util import normalize
from util import normalize_title
from util import safe_commit
from webpage import PublisherWebpage


def build_new_pub(doi, crossref_api):
    my_pub = Pub(id=doi, crossref_api_raw_new=crossref_api)
    my_pub.title = my_pub.crossref_title
    my_pub.normalized_title = normalize_title(my_pub.title)
    return my_pub


def add_new_pubs(pubs_to_commit):
    if not pubs_to_commit:
        return []

    pubs_indexed_by_id = dict((my_pub.id, my_pub) for my_pub in pubs_to_commit)
    ids_already_in_db = [
        id_tuple[0] for id_tuple in db.session.query(Pub.id).filter(Pub.id.in_(pubs_indexed_by_id.keys())).all()
    ]
    pubs_to_add_to_db = []

    for (pub_id, my_pub) in pubs_indexed_by_id.iteritems():
        if pub_id in ids_already_in_db:
            # merge if we need to
            pass
        else:
            pubs_to_add_to_db.append(my_pub)
            # logger.info(u"adding new pub {}".format(my_pub.id))

    if pubs_to_add_to_db:
        logger.info(u"adding {} pubs".format(len(pubs_to_add_to_db)))
        db.session.add_all(pubs_to_add_to_db)
        safe_commit(db)
    return pubs_to_add_to_db


def call_targets_in_parallel(targets):
    if not targets:
        return

    # logger.info(u"calling", targets)
    threads = []
    for target in targets:
        process = Thread(target=target, args=[])
        process.start()
        threads.append(process)
    for process in threads:
        try:
            process.join(timeout=60*10)
        except (KeyboardInterrupt, SystemExit):
            pass
        except Exception as e:
            logger.exception(u"thread Exception {} in call_targets_in_parallel. continuing.".format(e))
    # logger.info(u"finished the calls to {}".format(targets))


def call_args_in_parallel(target, args_list):
    # logger.info(u"calling", targets)
    threads = []
    for args in args_list:
        process = Thread(target=target, args=args)
        process.start()
        threads.append(process)
    for process in threads:
        try:
            process.join(timeout=60*10)
        except (KeyboardInterrupt, SystemExit):
            pass
        except Exception as e:
            logger.exception(u"thread Exception {} in call_args_in_parallel. continuing.".format(e))
    # logger.info(u"finished the calls to {}".format(targets))


def lookup_product_by_doi(doi):
    biblio = {"doi": doi}
    return lookup_product(**biblio)


def lookup_product(**biblio):
    my_pub = None
    if "doi" in biblio and biblio["doi"]:
        doi = clean_doi(biblio["doi"])

        # map unregistered JSTOR DOIs to real articles
        # for example https://www.jstor.org/stable/2244328?seq=1 says 10.2307/2244328 on the page
        # but https://doi.org/10.2307/2244328 goes nowhere and the article is at https://doi.org/10.1214/aop/1176990626

        jstor_overrides = {
            '10.2307/2244328': '10.1214/aop/1176990626',  # https://www.jstor.org/stable/2244328
            '10.2307/25151720': '10.1287/moor.1060.0190',  # https://www.jstor.org/stable/25151720
            '10.2307/2237638': '10.1214/aoms/1177704711',  # https://www.jstor.org/stable/2237638
        }

        doi = jstor_overrides.get(doi, doi)

        my_pub = Pub.query.get(doi)

        if my_pub:
            # logger.info(u"found {} in pub db table!".format(my_pub.id))
            my_pub.reset_vars()
        else:
            raise NoDoiException
        #     my_pub = Crossref(**biblio)
        #     logger.info(u"didn't find {} in crossref db table".format(my_pub))

    return my_pub


def refresh_pub(my_pub, do_commit=False):
    my_pub.run_with_hybrid()
    db.session.merge(my_pub)
    if do_commit:
        safe_commit(db)
    return my_pub


def thread_result_wrapper(func, args, res):
    res.append(func(*args))


# get rid of this when we get rid of POST endpoint
# for now, simplify it so it just calls the single endpoint
{'fraction_non_alphanumeric': 0.08270835533960072, 'fraction_numerical': 0.018590894686806803, 'mean_word_length': 3.3016810540663335, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 4, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4448370', 'n_tokens_mistral': 3264, 'n_tokens_neox': 3036, 'n_words': 777}