<commit_before>import os
import sys
import re
import hashlib
import json
import requests
import socket
import boto
import requests
from requests.auth import HTTPProxyAuth
from requests.packages.urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter
from time import time
from time import sleep

from app import requests_cache_bucket
from util import clean_doi
from util import get_tree
from util import get_link_target
from util import elapsed

MAX_PAYLOAD_SIZE_BYTES = 1000*1000*10 # 10mb
CACHE_FOLDER_NAME = "tng-requests-cache"


class CachedResponse:
    def __init__(self, **kwargs):
        self.headers = {}
        self.status_code = 200
        for (k, v) in kwargs.iteritems():
            setattr(self, k, v)

    @property
    def content(self):
        return self.file_contents

    @property
    def text(self):
        return self.file_contents

    # allows it to be treated the same way as a streaming response object
    def close(self):
        pass


def is_response_too_large(r):
    if not "Content-Length" in r.headers:
        # print u"can't tell if page is too large, no Content-Length header {}".format(r.url)
        return False

    content_length = r.headers["Content-Length"]
    # if is bigger than 1 MB, don't keep it don't parse it, act like we couldn't get it
    # if doing 100 in parallel, this would be 100MB, which fits within 512MB dyno limit
    if int(content_length) >= (1 * 1000 * 1000):
        print u"Content Too Large on GET on {url}".format(url=r.url)
        return True
    return False

# 10.2514/6.2006-5946!  https://arc.aiaa.org/doi/pdf/10.2514/6.2006-5946
# 10.3410/f.6269956.7654055 none
# 10.2514/6.2006-2106 none  (lots of redirects)
# 10.5040/9780567662088.part-003 none (book)
# 10.1016/j.jvcir.2016.03.027 (elsevier, relative links)
# 10.1002/(sici)1096-911x(200006)34:6<432::aid-mpo10>3.0.co;2-1 (has a blank tdm_api)

# python update.py Crossref.run_with_hybrid --id=10.2514/6.2006-5946

def get_crossref_resolve_url(url, related_pub=None):
    doi = clean_doi(url)

    if related_pub and related_pub.tdm_api:
        page = related_pub.tdm_api.encode("utf-8")
        # print "got doi tdm page from db"
    else:
        print "didn't find doi tdm page in db"
        # reset this in case it had been set
        os.environ["HTTP_PROXY"] = ""

        headers = {"Accept": "application/vnd.crossref.unixsd+xml"}
        headers["User-Agent"] = "oaDOI.org"
        headers["From"] = "<EMAIL>"

        connect_timeout = 600
        read_timeout = 600
        url = url.replace("http://", "https://")
        proxy_url = os.getenv("STATIC_IP_PROXY")
        static_ip_proxies = {"https": proxy_url, "http": proxy_url}
        r = requests.get(url,
                         headers=headers,
                         proxies=static_ip_proxies,
                         timeout=(connect_timeout, read_timeout),
                         allow_redirects=True,
                         verify=False
                         )
        if r and not r.encoding:
            r.encoding = "utf-8"

        if (r.status_code != 200) or len(r.content) == 0:
            # print u"r.status_code: {}".format(r.status_code)
            print u"WARNING: no crossref tdm_api for {}, so using resolve url".format(url)
            r = requests.get("http://doi.org/{}".format(doi),
                            allow_redirects=False,
                            timeout=(connect_timeout, read_timeout))
            # print u"new responses"
            # print u"r.status_code: {}".format(r.status_code)
            # print u"r.headers: {}".format(r.headers)
            response_url = r.headers["Location"]
            return response_url
        else:
            page = r.content
            if related_pub:
                related_pub.tdm_api = page  #archive it for later

    tree = get_tree(page)
    publication_type = tree.xpath("//doi/@type")[0]
    # print "publication_type", publication_type
    doi_data_stuff = tree.xpath("//doi_record//doi_data/resource/text()".format(publication_type))
    # print "doi_data_stuff", doi_data_stuff
    # this is ugly, but it works for now.  the last resolved one is the one we want.
    response_url = doi_data_stuff[-1]

    return response_url

def get_crawlera_session_id():
    # set up proxy
    saved_http_proxy = os.getenv("HTTP_PROXY", "")
    os.environ["HTTP_PROXY"] = "http://{}:DUMMY@proxy.crawlera.com:8010".format(os.getenv("CRAWLERA_KEY"))

    headers = {"X-Crawlera-Session": "create"}<commit_msg>handle bad tdm responses<commit_after>import os
import sys
import re
import hashlib
import json
import requests
import socket
import boto
import requests
from requests.auth import HTTPProxyAuth
from requests.packages.urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter
from time import time
from time import sleep

from app import requests_cache_bucket
from util import clean_doi
from util import get_tree
from util import get_link_target
from util import elapsed

MAX_PAYLOAD_SIZE_BYTES = 1000*1000*10 # 10mb
CACHE_FOLDER_NAME = "tng-requests-cache"


class CachedResponse:
    def __init__(self, **kwargs):
        self.headers = {}
        self.status_code = 200
        for (k, v) in kwargs.iteritems():
            setattr(self, k, v)

    @property
    def content(self):
        return self.file_contents

    @property
    def text(self):
        return self.file_contents

    # allows it to be treated the same way as a streaming response object
    def close(self):
        pass


def is_response_too_large(r):
    if not "Content-Length" in r.headers:
        # print u"can't tell if page is too large, no Content-Length header {}".format(r.url)
        return False

    content_length = r.headers["Content-Length"]
    # if is bigger than 1 MB, don't keep it don't parse it, act like we couldn't get it
    # if doing 100 in parallel, this would be 100MB, which fits within 512MB dyno limit
    if int(content_length) >= (1 * 1000 * 1000):
        print u"Content Too Large on GET on {url}".format(url=r.url)
        return True
    return False

# 10.2514/6.2006-5946!  https://arc.aiaa.org/doi/pdf/10.2514/6.2006-5946
# 10.3410/f.6269956.7654055 none
# 10.2514/6.2006-2106 none  (lots of redirects)
# 10.5040/9780567662088.part-003 none (book)
# 10.1016/j.jvcir.2016.03.027 (elsevier, relative links)
# 10.1002/(sici)1096-911x(200006)34:6<432::aid-mpo10>3.0.co;2-1 (has a blank tdm_api)

# python update.py Crossref.run_with_hybrid --id=10.2514/6.2006-5946

def get_crossref_resolve_url(url, related_pub=None):
    doi = clean_doi(url)

    if related_pub and related_pub.tdm_api:
        page = related_pub.tdm_api.encode("utf-8")
        # print "got doi tdm page from db"
    else:
        print "didn't find doi tdm page in db"
        # reset this in case it had been set
        os.environ["HTTP_PROXY"] = ""

        headers = {"Accept": "application/vnd.crossref.unixsd+xml"}
        headers["User-Agent"] = "oaDOI.org"
        headers["From"] = "<EMAIL>"

        connect_timeout = 600
        read_timeout = 600
        url = url.replace("http://", "https://")
        proxy_url = os.getenv("STATIC_IP_PROXY")
        static_ip_proxies = {"https": proxy_url, "http": proxy_url}
        r = requests.get(url,
                         headers=headers,
                         proxies=static_ip_proxies,
                         timeout=(connect_timeout, read_timeout),
                         allow_redirects=True,
                         verify=False
                         )
        if r and not r.encoding:
            r.encoding = "utf-8"

        if (r.status_code != 200) or len(r.content) == 0:
            # print u"r.status_code: {}".format(r.status_code)
            print u"WARNING: no crossref tdm_api for {}, so using resolve url".format(url)
            r = requests.get("http://doi.org/{}".format(doi),
                            allow_redirects=False,
                            timeout=(connect_timeout, read_timeout))
            # print u"new responses"
            # print u"r.status_code: {}".format(r.status_code)
            # print u"r.headers: {}".format(r.headers)
            response_url = r.headers["Location"]
            return response_url
        else:
            page = r.content
            if related_pub:
                related_pub.tdm_api = page  #archive it for later

    tree = get_tree(page)

    try:
        publication_type = tree.xpath("//doi/@type")[0]
    except IndexError:
        print u"didn't get a parsable crossref tdm page, so returning resolved url"
        return r.url
    # print "publication_type", publication_type
    doi_data_stuff = tree.xpath("//doi_record//doi_data/resource/text()".format(publication_type))
    # print "doi_data_stuff", doi_data_stuff
    # this is ugly, but it works for now.  the last resolved one is the one we want.
    response_url = doi_data_stuff[-1]

    return response_url

def get_crawlera_session_id():
    # set up proxy
    saved_http_proxy = os.getenv("HTTP_PROXY", "")
    os.environ["HTTP_PROXY"] = "http://{}:DUMMY@proxy.crawlera.com:8010".format(os.getenv("CRAWLERA_KEY"))

    headers = {"X-Crawlera-Session": "create"}
{'fraction_non_alphanumeric': 0.1087216248506571, 'fraction_numerical': 0.04366243075920495, 'mean_word_length': 2.5185326709973253, 'pattern_counts': {'":': 8, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4573641', 'n_tokens_mistral': 3261, 'n_tokens_neox': 2910, 'n_words': 755}