<commit_before>"""
Harvester for UTAustin for the SHARE project
"""

from __future__ import unicode_literals

from scrapi.base import OAIHarvester


class UTAustinHarvester(OAIHarvester):
    short_name = 'utaustin'
    long_name = 'University of Texas at Austin Digital Repository'
    url = 'https://repositories.lib.utexas.edu'

    base_url = 'http://repositories.lib.utexas.edu/oai/request'
    property_list = [
        'type', 'source', 'format', 'date',
        'identifier', 'setSpec', 'rights'
    ]
    approved_sets = [
        'hdl_2152_1',
        'hdl_2152_13541',
        'hdl_2152_22957',
        'hdl_2152_13341',
        'hdl_2152_11183',
        'hdl_2152_15554',
        'hdl_2152_21116',
        'hdl_2152_11227',
        'hdl_2152_26',
        'hdl_2152_25673',
        'hdl_2152_21442',
        'hdl_2152_11019',
        'hdl_2152_10079',
        'hdl_2152_23952',
        'hdl_2152_19781',
        'hdl_2152_4',
        'hdl_2152_5',
        'hdl_2152_15265',
        'hdl_2152_20099',
        'hdl_2152_4027',
        'hdl_2152_22392',
        'hdl_2152_24880',
        'hdl_2152_24538',
        'hdl_2152_20329',
        'hdl_2152_14283',
        'hdl_2152_14697',
        'hdl_2152_16482',
        'hdl_2152_24831',
        'hdl_2152_11681',
        'hdl_2152_15722',
        'hdl_2152_7103',
        'hdl_2152_20398',
        'hdl_2152_7100',
        'hdl_2152_7105',
        'hdl_2152_7102',
        'hdl_2152_7101',
        'hdl_2152_17706',
        'hdl_2152_15040',
        'hdl_2152_14309',
        'hdl_2152_18015',
        'hdl_2152_6854',
        'hdl_2152_6851',
        'hdl_2152_15082'
    ]
<commit_msg>Change base OAI Url<commit_after>"""
Harvester for UTAustin for the SHARE project
"""

from __future__ import unicode_literals

from scrapi.base import OAIHarvester


class UTAustinHarvester(OAIHarvester):
    short_name = 'utaustin'
    long_name = 'University of Texas at Austin Digital Repository'
    url = 'https://repositories.lib.utexas.edu'

    base_url = 'https://repositories.lib.utexas.edu/utexas-oai/request'
    property_list = [
        'type', 'source', 'format', 'date',
        'identifier', 'setSpec', 'rights'
    ]
    approved_sets = [
        'hdl_2152_1',
        'hdl_2152_13541',
        'hdl_2152_22957',
        'hdl_2152_13341',
        'hdl_2152_11183',
        'hdl_2152_15554',
        'hdl_2152_21116',
        'hdl_2152_11227',
        'hdl_2152_26',
        'hdl_2152_25673',
        'hdl_2152_21442',
        'hdl_2152_11019',
        'hdl_2152_10079',
        'hdl_2152_23952',
        'hdl_2152_19781',
        'hdl_2152_4',
        'hdl_2152_5',
        'hdl_2152_15265',
        'hdl_2152_20099',
        'hdl_2152_4027',
        'hdl_2152_22392',
        'hdl_2152_24880',
        'hdl_2152_24538',
        'hdl_2152_20329',
        'hdl_2152_14283',
        'hdl_2152_14697',
        'hdl_2152_16482',
        'hdl_2152_24831',
        'hdl_2152_11681',
        'hdl_2152_15722',
        'hdl_2152_7103',
        'hdl_2152_20398',
        'hdl_2152_7100',
        'hdl_2152_7105',
        'hdl_2152_7102',
        'hdl_2152_7101',
        'hdl_2152_17706',
        'hdl_2152_15040',
        'hdl_2152_14309',
        'hdl_2152_18015',
        'hdl_2152_6854',
        'hdl_2152_6851',
        'hdl_2152_15082'
    ]

{'fraction_non_alphanumeric': 0.11720169594185342, 'fraction_numerical': 0.2204724409448819, 'mean_word_length': 2.2541871921182266, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4013938', 'n_tokens_mistral': 1806, 'n_tokens_neox': 1391, 'n_words': 171}