<commit_before>import os
import re
import requests
from time import time
from Levenshtein import ratio
from collections import defaultdict
from HTMLParser import HTMLParser

from webpage import PublisherWebpage, WebpageInOpenRepo, WebpageInUnknownRepo
from oa_local import find_normalized_license
from util import elapsed
from util import normalize


DEBUG_BASE = True


def get_fulltext_webpages_from_our_base_doc(doc):
    response = []

    license = doc.get("fulltext_license", None)

    # workaround for a bug there was in the normalized license
    license_string_in_doc = doc.get("license", "")
    if license_string_in_doc:
        if "orks not in the public domain" in license_string_in_doc:
            license = None
        if not license:
            license = find_normalized_license(license_string_in_doc)<commit_msg>make sure not scraping by accident, change user-agent to oaDOI.org<commit_after>import os
import re
import requests
from time import time
from Levenshtein import ratio
from collections import defaultdict
from HTMLParser import HTMLParser

from webpage import PublisherWebpage, WebpageInOpenRepo, WebpageInUnknownRepo
from oa_local import find_normalized_license
from util import elapsed
from util import normalize


DEBUG_BASE = True

# don't scrape these ever.  from email:
# 1.       http://elar.rsvpu.ru 100% open access
# 2.       http://elib.uraic.ru 100% open access
# 3.       http://elar.usfeu.ru 100% open access
# 4.       http://elar.urfu.ru 99% open access
# 5.       http://elar.uspu.ru 75% open access (this community is closed - http://elar.uspu.ru/handle/uspu/2914)


def get_fulltext_webpages_from_our_base_doc(doc):
    response = []

    license = doc.get("fulltext_license", None)

    # workaround for a bug there was in the normalized license
    license_string_in_doc = doc.get("license", "")
    if license_string_in_doc:
        if "orks not in the public domain" in license_string_in_doc:
            license = None
        if not license:
            license = find_normalized_license(license_string_in_doc)
{'fraction_non_alphanumeric': 0.0648282535074988, 'fraction_numerical': 0.010643444605708756, 'mean_word_length': 3.7981438515081205, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4277197', 'n_tokens_mistral': 640, 'n_tokens_neox': 599, 'n_words': 200}