<commit_before>import os
import requests
from time import time
from Levenshtein import ratio
from collections import defaultdict

from webpage import PublisherWebpage, WebpageInOpenRepo, WebpageInUnknownRepo
from util import elapsed
from util import normalize


DEBUG_BASE = False


def get_urls_from_our_base_doc(doc):
    response = []

    if "urls" in doc:
        # pmc can only add pmc urls.  otherwise has junk about dois that aren't actually open.
        if u"PubMed Central (PMC)" in doc["sources"]:
            for url in doc["urls"]:
                if "/pmc/" in url and url != "http://www.ncbi.nlm.nih.gov/pmc/articles/PMC":
                    response += [url]
        else:
            response += doc["urls"]

    # oxford IR doesn't return URLS, instead it returns IDs from which we can build URLs
    # example: https://www.base-search.net/Record/5c1cf4038958134de9700b6144ae6ff9e78df91d3f8bbf7902cb3066512f6443/
    if "sources" in doc and "Oxford University Research Archive (ORA)" in doc["sources"]:
        if "relations" in doc:
            for relation in doc["relations"]:
                if relation.startswith("uuid"):
                    response += [u"https://ora.ox.ac.uk/objects/{}".format(relation)]

    return response



def normalize_title_for_querying(title):
    if not title:
        return ""
    title_to_query = ""
    title = title.lower()
    # can't just replace all punctuation because ' replaced with ? gets no hits
    title = title.replace('"', "?")
    title = title.replace('#', "?")
    title = title.replace('=', "?")
    title = title.replace('&', "?")
    title = title.replace('%', "?")
    title = title.replace('/', "?")
    # title = title.replace('-', "*")

    # only bother looking up titles that are at least 3 words long
    title_words = title.split()
    if len(title_words) >= 3:
        # only look up the first 12 words
        title_to_query = u" ".join(title_words[0:12])

    return title_to_query



def call_our_base(my_pub):
    if not my_pub:
        return

    webpages_to_return = []
    found_a_base1 = False
    title = my_pub.best_title
    title_to_query = normalize_title_for_querying(title)

    if not title_to_query:
        return webpages_to_return

    # now do the lookup in base
    # query_string = u'title=%22{}%22'.format(title_to_query)
    query_string = u'title:({})'.format(title_to_query)

    if my_pub.first_author_lastname:
        query_string += u" AND authors:({})".format(my_pub.first_author_lastname)

    # if my_product.doi:
    #     query_string += u" OR urls={}".format(my_product.doi)

    url_template = u"{base_url}/base/_search?pretty&size=20&q={query_string}"
    url = url_template.format(base_url=os.getenv("BASE_URL"), query_string=query_string)

    if DEBUG_BASE:
        print u"calling our base with {}\n".format(url)

    start_time = time()
    r = None
    try:
        r = requests.get(url, timeout=10)
        # print u"** querying with {} titles took {}s".format(len(titles), elapsed(start_time))
    except requests.exceptions.ConnectionError:
        print u"connection error in call_our_base, skipping."
    except requests.Timeout:
        print u"TIMEOUT error in call_our_base, skipping."

    if r != None and r.status_code != 200:
        print u"problem searching base! status_code={}".format(r.status_code)<commit_msg>replace ! in lookup, and look up author normalized<commit_after>import os
import requests
from time import time
from Levenshtein import ratio
from collections import defaultdict

from webpage import PublisherWebpage, WebpageInOpenRepo, WebpageInUnknownRepo
from util import elapsed
from util import normalize


DEBUG_BASE = False


def get_urls_from_our_base_doc(doc):
    response = []

    if "urls" in doc:
        # pmc can only add pmc urls.  otherwise has junk about dois that aren't actually open.
        if u"PubMed Central (PMC)" in doc["sources"]:
            for url in doc["urls"]:
                if "/pmc/" in url and url != "http://www.ncbi.nlm.nih.gov/pmc/articles/PMC":
                    response += [url]
        else:
            response += doc["urls"]

    # oxford IR doesn't return URLS, instead it returns IDs from which we can build URLs
    # example: https://www.base-search.net/Record/5c1cf4038958134de9700b6144ae6ff9e78df91d3f8bbf7902cb3066512f6443/
    if "sources" in doc and "Oxford University Research Archive (ORA)" in doc["sources"]:
        if "relations" in doc:
            for relation in doc["relations"]:
                if relation.startswith("uuid"):
                    response += [u"https://ora.ox.ac.uk/objects/{}".format(relation)]

    return response



def normalize_title_for_querying(title):
    if not title:
        return ""
    title_to_query = ""
    title = title.lower()
    # can't just replace all punctuation because ' replaced with ? gets no hits
    title = title.replace('"', "?")
    title = title.replace('#', "?")
    title = title.replace('=', "?")
    title = title.replace('&', "?")
    title = title.replace('%', "?")
    title = title.replace('/', "?")
    title = title.replace('!', "?")
    # title = title.replace('-', "*")

    # only bother looking up titles that are at least 3 words long
    title_words = title.split()
    if len(title_words) >= 3:
        # only look up the first 12 words
        title_to_query = u" ".join(title_words[0:12])

    return title_to_query



def call_our_base(my_pub):
    if not my_pub:
        return

    webpages_to_return = []
    found_a_base1 = False
    title = my_pub.best_title
    title_to_query = normalize_title_for_querying(title)

    if not title_to_query:
        return webpages_to_return

    # now do the lookup in base
    # query_string = u'title=%22{}%22'.format(title_to_query)
    query_string = u'title:({})'.format(title_to_query)

    if my_pub.first_author_lastname:
        query_string += u" AND authors:({} OR {})".format(
            my_pub.first_author_lastname, normalize(my_pub.first_author_lastname))

    # if my_product.doi:
    #     query_string += u" OR urls={}".format(my_product.doi)

    url_template = u"{base_url}/base/_search?pretty&size=20&q={query_string}"
    url = url_template.format(base_url=os.getenv("BASE_URL"), query_string=query_string)

    if DEBUG_BASE:
        print u"calling our base with {}\n".format(url)

    start_time = time()
    r = None
    try:
        r = requests.get(url, timeout=10)
        # print u"** querying with {} titles took {}s".format(len(titles), elapsed(start_time))
    except requests.exceptions.ConnectionError:
        print u"connection error in call_our_base, skipping."
    except requests.Timeout:
        print u"TIMEOUT error in call_our_base, skipping."

    if r != None and r.status_code != 200:
        print u"problem searching base! status_code={}".format(r.status_code)
{'fraction_non_alphanumeric': 0.11830780452224654, 'fraction_numerical': 0.018088986141502554, 'mean_word_length': 3.0117027501462843, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1811924', 'n_tokens_mistral': 2299, 'n_tokens_neox': 2147, 'n_words': 568}