<commit_before># coding=utf-8
from __future__ import print_function
from __future__ import unicode_literals
import argparse
import logging.handlers
from io import open
import io

import requests
import sys
from requests import Session
from requests.exceptions import HTTPError
from six.moves import configparser
from tqdm import tqdm
from prompter import yesno

try:
    # Use lxml if installed, since it's faster ...
    from lxml import etree
except ImportError:
    # ... but also support standard ElementTree, since installation of lxml can be cumbersome
    import xml.etree.ElementTree as etree

logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
logging.getLogger('requests').setLevel(logging.WARNING)
formatter = logging.Formatter('[%(asctime)s %(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%I:%S')

console_handler = logging.StreamHandler()
console_handler.setFormatter(formatter)
logger.addHandler(console_handler)

log_capture_string = io.StringIO()
capture_handler = logging.StreamHandler(log_capture_string)
capture_handler.setLevel(logging.DEBUG)
capture_handler.setFormatter(formatter)
logger.addHandler(capture_handler)

nsmap = {
    'e20': 'http://explain.z3950.org/dtd/2.0/',
    'e21': 'http://explain.z3950.org/dtd/2.1/',
    'srw': 'http://www.loc.gov/zing/srw/',
    'diag': 'http://www.loc.gov/zing/srw/diagnostic/',
}


class SruErrorResponse(RuntimeError):
    pass


def normalize_term(term):
    # Normalize term so it starts with a capital letter. If the term is a subject string
    # fused by " : ", normalize all components.
    if term is None or len(term) == 0:
        return term

    return ' : '.join([component[0].upper() + component[1:] for component in term.strip().split(' : ')])


def subject_fields(marc_record, term, vocabulary, tags, exact_only=False):
    """
    For a given MARC record, return subject fields matching the vocabulary and term.
    :param marc_record:
    :param term:
    :param vocabulary:
    :param tags:
    :param exact_only: True to only return fields with the term in $a and with no $x
                       False to return fields with the term in either $a or $x
    :return:
    """

    fields = []
    for tag in tags:
        for field in marc_record.findall('./datafield[@tag="{}"]'.format(tag)):
            if field.findtext('subfield[@code="2"]') != vocabulary:
                # Wrong vocabulary
                continue

            sfa = normalize_term(field.findtext('subfield[@code="a"]'))
            sfx = normalize_term(field.findtext('subfield[@code="x"]'))

            components = term.split(' : ')
            if len(components) == 2:
                if sfa == components[0] and sfx == components[1]:
                    fields.append(field)
            elif exact_only:
                if sfa == term and sfx is None:
                    fields.append(field)
            else:
                if sfa == term or sfx == term:
                    fields.append(field)

    return fields


def sru_search(query, url):
    # A SRU search generator that returns MARC records
    start_record = 1
    while True:
        response = requests.get(url, params={
            'version': '1.2',
            'operation': 'searchRetrieve',
            'startRecord': start_record,
            'maximumRecords': '50',
            'query': query,
        })
        root = etree.fromstring(response.text.encode('utf-8'))  # Takes ~ 4 seconds for 50 records!

        for diagnostic in root.findall('srw:diagnostics/diag:diagnostic', namespaces=nsmap):
            raise SruErrorResponse(diagnostic.findtext('diag:message', namespaces=nsmap))

        num_records = root.findtext('srw:numberOfRecords', namespaces=nsmap)
        for record in root.iterfind('srw:records/srw:record', namespaces=nsmap):
            record_no = record.findtext('srw:recordPosition', namespaces=nsmap)
            yield int(record_no), int(num_records), record.find('srw:recordData/record', namespaces=nsmap)

        nrp = root.find('srw:nextRecordPosition', namespaces=nsmap)
        if nrp is not None:
            start_record = nrp.text
        else:
            break  # Enden er nær, den er faktisk her!


class Bib(object):

    def __init__(self, alma, doc):
        self.alma = alma
        self.doc = doc
        self.mms_id = self.doc.findtext('mms_id')
        self.marc_record = self.doc.find('record')

    def remove_duplicate_fields(self, vocabulary, term, tags):
        strenger = []
        for field in subject_fields(self.marc_record, vocabulary=vocabulary, term=term, tags=tags):
            streng = [field.get('tag')]
            for subfield in field.findall('subfield'):
                if subfield.get('code') in ['a', 'x']:
                    streng.append(subfield.text)
                elif subfield.get('code') not in ['2', '0']:
                    logger.info('Emnefeltet inneholdt uventede delfelt: %s', etree.tostring(subfield))
            if streng in strenger:
                logger.info('Fjerner duplikat emnefelt: "%s" ', ' : '.join(streng))
                self.marc_record.remove(field)
                continue
            strenger.append(streng)

    def edit_subject(self, vocabulary, old_term, new_term, tags):
        self.remove_duplicate_fields(vocabulary, old_term, tags)

        old_term_comp = old_term.split(' : ')
        new_term_comp = new_term.split(' : ')

        for field in subject_fields(self.marc_record, vocabulary=vocabulary, term=old_term, tags=tags):
            sfa = field.find('subfield[@code="a"]')
            sfx = field.find('subfield[@code="x"]')
            sfa_m0 = sfa is not None and normalize_term(sfa.text) == old_term_comp[0]
            sfx_m0 = sfx is not None and normalize_term(sfx.text) == old_term_comp[0]
            if len(old_term_comp) == 2:
                sfx_m1 = sfx is not None and normalize_term(sfx.text) == old_term_comp[1]
                if sfa_m0 and sfx_m1:
                    if len(new_term_comp) == 2:
                        sfa.text = new_term_comp[0]
                        sfx.text = new_term_comp[1]
                    else:
                        sfa.text = new_term_comp[0]
                        field.remove(sfx)
            else:
                if sfa_m0:
                    sfa.text = new_term_comp[0]
                elif sfx_m0:
                    sfx.text = new_term_comp[0]

        return self  # for chaining

    def remove_subject(self, vocabulary, term, tags):
        for field in subject_fields(self.marc_record, vocabulary=vocabulary, term=term, tags=tags, exact_only=True):
            self.marc_record.remove(field)
        return self  # for chaining

    def save(self):
        try:
            self.alma.put('/bibs/{}'.format(self.mms_id),
                          data=etree.tostring(self.doc),
                          headers={'Content-Type': 'application/xml'})
        except HTTPError as error:
            raise RuntimeError('Failed to save record. Status: %s. Response: %s'
                               % (error.response.status_code, error.response.text))


class Alma(object):

    def __init__(self, api_region, api_key):<commit_msg>[#1] Be om bekreftelse hvis post er lenket til CZ<commit_after># coding=utf-8
from __future__ import print_function
from __future__ import unicode_literals
import argparse
import logging.handlers
from io import open
import io

import requests
import sys
from requests import Session
from requests.exceptions import HTTPError
from six.moves import configparser
from tqdm import tqdm
from prompter import yesno

try:
    # Use lxml if installed, since it's faster ...
    from lxml import etree
except ImportError:
    # ... but also support standard ElementTree, since installation of lxml can be cumbersome
    import xml.etree.ElementTree as etree

logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
logging.getLogger('requests').setLevel(logging.WARNING)
formatter = logging.Formatter('[%(asctime)s %(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%I:%S')

console_handler = logging.StreamHandler()
console_handler.setFormatter(formatter)
logger.addHandler(console_handler)

log_capture_string = io.StringIO()
capture_handler = logging.StreamHandler(log_capture_string)
capture_handler.setLevel(logging.DEBUG)
capture_handler.setFormatter(formatter)
logger.addHandler(capture_handler)

nsmap = {
    'e20': 'http://explain.z3950.org/dtd/2.0/',
    'e21': 'http://explain.z3950.org/dtd/2.1/',
    'srw': 'http://www.loc.gov/zing/srw/',
    'diag': 'http://www.loc.gov/zing/srw/diagnostic/',
}


class SruErrorResponse(RuntimeError):
    pass


def normalize_term(term):
    # Normalize term so it starts with a capital letter. If the term is a subject string
    # fused by " : ", normalize all components.
    if term is None or len(term) == 0:
        return term

    return ' : '.join([component[0].upper() + component[1:] for component in term.strip().split(' : ')])


def subject_fields(marc_record, term, vocabulary, tags, exact_only=False):
    """
    For a given MARC record, return subject fields matching the vocabulary and term.
    :param marc_record:
    :param term:
    :param vocabulary:
    :param tags:
    :param exact_only: True to only return fields with the term in $a and with no $x
                       False to return fields with the term in either $a or $x
    :return:
    """

    fields = []
    for tag in tags:
        for field in marc_record.findall('./datafield[@tag="{}"]'.format(tag)):
            if field.findtext('subfield[@code="2"]') != vocabulary:
                # Wrong vocabulary
                continue

            sfa = normalize_term(field.findtext('subfield[@code="a"]'))
            sfx = normalize_term(field.findtext('subfield[@code="x"]'))

            components = term.split(' : ')
            if len(components) == 2:
                if sfa == components[0] and sfx == components[1]:
                    fields.append(field)
            elif exact_only:
                if sfa == term and sfx is None:
                    fields.append(field)
            else:
                if sfa == term or sfx == term:
                    fields.append(field)

    return fields


def sru_search(query, url):
    # A SRU search generator that returns MARC records
    start_record = 1
    while True:
        response = requests.get(url, params={
            'version': '1.2',
            'operation': 'searchRetrieve',
            'startRecord': start_record,
            'maximumRecords': '50',
            'query': query,
        })
        root = etree.fromstring(response.text.encode('utf-8'))  # Takes ~ 4 seconds for 50 records!

        for diagnostic in root.findall('srw:diagnostics/diag:diagnostic', namespaces=nsmap):
            raise SruErrorResponse(diagnostic.findtext('diag:message', namespaces=nsmap))

        num_records = root.findtext('srw:numberOfRecords', namespaces=nsmap)
        for record in root.iterfind('srw:records/srw:record', namespaces=nsmap):
            record_no = record.findtext('srw:recordPosition', namespaces=nsmap)
            yield int(record_no), int(num_records), record.find('srw:recordData/record', namespaces=nsmap)

        nrp = root.find('srw:nextRecordPosition', namespaces=nsmap)
        if nrp is not None:
            start_record = nrp.text
        else:
            break  # Enden er nær, den er faktisk her!


class Bib(object):

    def __init__(self, alma, doc):
        self.alma = alma
        self.doc = doc
        self.mms_id = self.doc.findtext('mms_id')
        self.marc_record = self.doc.find('record')
        self.linked_to_cz = self.doc.findtext('linked_record_id[@type="CZ"]') or None

    def remove_duplicate_fields(self, vocabulary, term, tags):
        strenger = []
        for field in subject_fields(self.marc_record, vocabulary=vocabulary, term=term, tags=tags):
            streng = [field.get('tag')]
            for subfield in field.findall('subfield'):
                if subfield.get('code') in ['a', 'x']:
                    streng.append(subfield.text)
                elif subfield.get('code') not in ['2', '0']:
                    logger.info('Emnefeltet inneholdt uventede delfelt: %s', etree.tostring(subfield))
            if streng in strenger:
                logger.info('Fjerner duplikat emnefelt: "%s" ', ' : '.join(streng))
                self.marc_record.remove(field)
                continue
            strenger.append(streng)

    def edit_subject(self, vocabulary, old_term, new_term, tags):
        self.remove_duplicate_fields(vocabulary, old_term, tags)

        old_term_comp = old_term.split(' : ')
        new_term_comp = new_term.split(' : ')

        for field in subject_fields(self.marc_record, vocabulary=vocabulary, term=old_term, tags=tags):
            sfa = field.find('subfield[@code="a"]')
            sfx = field.find('subfield[@code="x"]')
            sfa_m0 = sfa is not None and normalize_term(sfa.text) == old_term_comp[0]
            sfx_m0 = sfx is not None and normalize_term(sfx.text) == old_term_comp[0]
            if len(old_term_comp) == 2:
                sfx_m1 = sfx is not None and normalize_term(sfx.text) == old_term_comp[1]
                if sfa_m0 and sfx_m1:
                    if len(new_term_comp) == 2:
                        sfa.text = new_term_comp[0]
                        sfx.text = new_term_comp[1]
                    else:
                        sfa.text = new_term_comp[0]
                        field.remove(sfx)
            else:
                if sfa_m0:
                    sfa.text = new_term_comp[0]
                elif sfx_m0:
                    sfx.text = new_term_comp[0]

        return self  # for chaining

    def remove_subject(self, vocabulary, term, tags):
        for field in subject_fields(self.marc_record, vocabulary=vocabulary, term=term, tags=tags, exact_only=True):
            self.marc_record.remove(field)
        return self  # for chaining

    def save(self):
        if self.linked_to_cz:
            logger.info(' -> OBS! Posten er koblet til CZ! Koblingen blir brutt hvis du oppdaterer posten!')
            if yesno('Vil du fortsette allikevel?', default='no'):
                logger.info(' -> Hopper over denne posten')
                return
            logger.info(' -> Oppdaterer posten. Koblingen til CZ blir brutt.')

        try:
            self.alma.put('/bibs/{}'.format(self.mms_id),
                          data=etree.tostring(self.doc),
                          headers={'Content-Type': 'application/xml'})
        except HTTPError as error:
            raise RuntimeError('Failed to save record. Status: %s. Response: %s'
                               % (error.response.status_code, error.response.text))


class Alma(object):

    def __init__(self, api_region, api_key):
{'fraction_non_alphanumeric': 0.11379426644182125, 'fraction_numerical': 0.007082630691399663, 'mean_word_length': 2.3981205592482238, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '429663', 'n_tokens_mistral': 4623, 'n_tokens_neox': 4434, 'n_words': 1088}