<commit_before># coding=utf-8
from __future__ import unicode_literals

import io
import logging
from copy import copy
from datetime import datetime
from textwrap import dedent

from prompter import yesno
from requests.exceptions import HTTPError
from tqdm import tqdm

from .skosmos import Skosmos
from .sru import TooManyResults
from .task import AddTask, ReplaceTask, MoveTask, DeleteTask

log = logging.getLogger(__name__)
formatter = logging.Formatter('[%(asctime)s %(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%I:%S')

log_capture_string = io.StringIO()
capture_handler = logging.StreamHandler(log_capture_string)
capture_handler.setLevel(logging.INFO)
capture_handler.setFormatter(formatter)
log.addHandler(capture_handler)


class Job(object):
    def __init__(self, action, source_concept, target_concepts=None, sru=None, alma=None, mailer=None):
        self.dry_run = False
        self.interactive = True
        self.show_progress = True
        self.show_diffs = False

        self.sru = sru
        self.alma = alma
        self.mailer = mailer

        self.action = action
        self.source_concept = source_concept
        self.target_concepts = target_concepts or []
        self.vocabulary = self.source_concept.vocabulary

        self.job_name = datetime.now().isoformat()
        self.skosmos = Skosmos(self.vocabulary.skosmos_code)

        if self.source_concept.tag == '648' and self.source_concept.vocabulary.marc_code == 'noubomn':
            raise RuntimeError('Editing 648 for noubomn is disabled until we get rid of the 650 duplicates')
            # log.info('Note: For the 648 field, we will also fix the 650 duplicate')

        self.authorize()
        self.steps = []
        self.generate_steps()

    @staticmethod
    def generate_replace_tasks(src, dst):

        if len(src.components) == 1 and len(dst.components) == 1:
            tasks = [ReplaceTask(src, dst, True)]

            src_copy = copy(src)
            dst_copy = copy(dst)
            src_copy.sf['x'] = src_copy.sf['a']
            del src_copy.sf['a']
            dst_copy.sf['x'] = dst_copy.sf['a']
            del dst_copy.sf['a']

            tasks.append(ReplaceTask(src_copy, dst_copy, True))
        else:
            tasks = [ReplaceTask(src, dst)]

        return tasks

    def generate_steps(self):

        if self.action == 'delete':
            # Delete
            self.steps.append(DeleteTask(self.source_concept))

        elif self.action == 'rename':
            src = self.source_concept
            dst = self.target_concepts[0]

            # Rename
            if src.term != dst.term:
                for step in self.generate_replace_tasks(src, dst):
                    self.steps.append(step)

                # Note: Update source concept before next step (move)
                dst_copy = copy(dst)
                dst_copy.tag = src.tag
                src = dst_copy

            # Move
            if src.tag != dst.tag:
                self.steps.append(MoveTask(src, dst.tag))

            # Add
            if len(self.target_concepts) > 1:
                self.steps.append(AddTask(self.target_concepts[1]))

    def update_record(self, bib):

        for step in self.steps:
            step.run(bib.marc_record)

        if bib.cz_link is not None:
            log.warning(dedent(
                '''\
                Encountered a Community Zone record. Updating such records through the API will
                currently cause them to be de-linked from CZ, which is probably not what you want.
                Until Ex Libris fixes this, you're best off editing the record manually in Alma.
                '''))

            if not self.interactive or yesno('Do you want to update the record and break CZ linkage?', default='no'):
                log.warning(' -> Skipping this record. You should update it manually in Alma!')
                return

            log.warning(' -> Updating the record. The CZ connection will be lost!')

        try:
            bib.save(self.show_diffs, self.dry_run)
        except HTTPError:
            msg = '*** Failed to save record %s --- Please try to edit the record manually in Alma ***'
            log.error(msg, bib.mms_id)

    def authorize(self):
        if self.action == 'delete':
            return
        self.source_concept.authorize(self.skosmos)
        self.target_concepts[0].authorize(self.skosmos)
        if self.target_concepts[0].sf['0'] is None:
            # Use the source term identifier (if we just moved a concept)
            self.target_concepts[0].sf['0'] = self.source_concept.sf['0']
        if self.target_concepts[0].sf['0'] is None:
            log.warning('Neither the source term nor the (first) target term could be authorized in Skosmos.')
        for target_concept in self.target_concepts[1:]:
            target_concept.authorize(self.skosmos)
            if target_concept.sf['0'] is None:
                log.warning('The target term "%s" could not be authorized in Skosmos.', target_concept)

    def start(self):

        if self.alma.name is not None:
            log.info('Alma environment: %s', self.alma.name)

        for n, step in enumerate(self.steps):
            log.info('Step %d of %d: %s', n + 1, len(self.steps), step)

        if self.dry_run:
            log.info('Dry run: No catalog records will be touched!')

        # ------------------------------------------------------------------------------------
        # Del 1: Søk mot SRU for å finne over alle bibliografiske poster med emneordet.
        # Vi må filtrere resultatlista i etterkant fordi
        #  - vi mangler en egen indeks for Realfagstermer, så vi må søke mot `alma.subjects`
        #  - søket er ikke presist, så f.eks. "Monstre" vil gi treff i "Mønstre"
        #
        # I fremtiden, når vi får $0 på alle poster, kan vi bruke indeksen `alma.authority_id`
        # i stedet.

        valid_records = []
        pbar = None
        cql_query = 'alma.subjects=="%s" AND alma.authority_vocabulary = "%s"' % (self.source_concept.term,
                                                                                  self.vocabulary.marc_code)

        try:
            for marc_record in self.sru.search(cql_query):
                if pbar is None and self.show_progress and self.sru.num_records > 50:
                    pbar = tqdm(total=self.sru.num_records, desc='Filtering SRU results')

                log.debug('Checking record %s', marc_record.id())

                for step in self.steps:
                    if step.match(marc_record):
                        valid_records.append(marc_record.id())
                        break

                if pbar is not None:
                    pbar.update()
            if pbar is not None:
                pbar.close()
        except TooManyResults:
            log.error('More than 10,000 results would have to be checked, but the Alma SRU service does ' +
                      'not allow us to retrieve more than 10,000 results. Annoying? Go vote for this:\n' +
                      'http://ideas.exlibrisgroup.com/forums/308173-alma/suggestions/' +
                      '18737083-sru-srw-increase-the-10-000-record-retrieval-limi')
            return []

        if len(valid_records) == 0:
            log.info('No matching catalog records found')
            return []
        else:
            log.info('%d catalog records will be updated', len(valid_records))

        if self.interactive and not yesno('Continue?', default='yes'):
            log.info('Bye')
            return []

        # ------------------------------------------------------------------------------------
        # Del 2: Nå har vi en liste over MMS-IDer for bibliografiske poster vi vil endre.
        # Vi går gjennom dem én for én, henter ut posten med Bib-apiet, endrer og poster tilbake.

        for n, mms_id in enumerate(valid_records):
            log.info(' %3d/%d: %s', n + 1, len(valid_records), mms_id)
            bib = self.alma.bibs(mms_id)
            self.update_record(bib)

        n_posts = '{:d} {}'.format(len(valid_records), 'record' if len(valid_records) == 1 else 'records')

        subject = None

        if self.action == 'delete':
            args = (self.source_concept.tag, self.source_concept.term, n_posts)
            subject = 'Deleted {} "{}" in {}'.format(*args)

        elif self.action == 'rename':
            if len(self.target_concepts) == 2:
                args = (self.source_concept.tag, self.source_concept.term,
                        self.target_concepts[0].tag, self.target_concepts[0].term,
                        self.target_concepts[1].tag, self.target_concepts[1].term,
                        n_posts)
                subject = 'Changed {} "{}" to {} "{}" + {} "{}" in {}'.format(*args)

            elif len(self.target_concepts) == 1:
                if self.source_concept.term == self.target_concepts[0].term:
                    args = (self.source_concept.tag, self.source_concept.term,
                            self.target_concepts[0].tag, n_posts)
                    subject = 'Moved {} "{}" to {} in {}'.format(*args)
                else:
                    args = (self.source_concept.tag, self.source_concept.term,
                            self.target_concepts[0].tag, self.target_concepts[0].term,
                            n_posts)
                    subject = 'Changed {} "{}" to {} "{}" in {}'.format(*args)

        body = log_capture_string.getvalue()

        if subject is not None:
            log.info(subject)
            if self.mailer is not None and not self.dry_run:
                self.mailer.send(subject, body)

        return valid_records
<commit_msg>Don't confirm to do nothing..<commit_after># coding=utf-8
from __future__ import unicode_literals

import io
import logging
from copy import copy
from datetime import datetime
from textwrap import dedent

from prompter import yesno
from requests.exceptions import HTTPError
from tqdm import tqdm

from .skosmos import Skosmos
from .sru import TooManyResults
from .task import AddTask, ReplaceTask, MoveTask, DeleteTask

log = logging.getLogger(__name__)
formatter = logging.Formatter('[%(asctime)s %(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%I:%S')

log_capture_string = io.StringIO()
capture_handler = logging.StreamHandler(log_capture_string)
capture_handler.setLevel(logging.INFO)
capture_handler.setFormatter(formatter)
log.addHandler(capture_handler)


class Job(object):
    def __init__(self, action, source_concept, target_concepts=None, sru=None, alma=None, mailer=None):
        self.dry_run = False
        self.interactive = True
        self.show_progress = True
        self.show_diffs = False

        self.sru = sru
        self.alma = alma
        self.mailer = mailer

        self.action = action
        self.source_concept = source_concept
        self.target_concepts = target_concepts or []
        self.vocabulary = self.source_concept.vocabulary

        self.job_name = datetime.now().isoformat()
        self.skosmos = Skosmos(self.vocabulary.skosmos_code)

        if self.source_concept.tag == '648' and self.source_concept.vocabulary.marc_code == 'noubomn':
            raise RuntimeError('Editing 648 for noubomn is disabled until we get rid of the 650 duplicates')
            # log.info('Note: For the 648 field, we will also fix the 650 duplicate')

        self.authorize()
        self.steps = []
        self.generate_steps()

    @staticmethod
    def generate_replace_tasks(src, dst):

        if len(src.components) == 1 and len(dst.components) == 1:
            tasks = [ReplaceTask(src, dst, True)]

            src_copy = copy(src)
            dst_copy = copy(dst)
            src_copy.sf['x'] = src_copy.sf['a']
            del src_copy.sf['a']
            dst_copy.sf['x'] = dst_copy.sf['a']
            del dst_copy.sf['a']

            tasks.append(ReplaceTask(src_copy, dst_copy, True))
        else:
            tasks = [ReplaceTask(src, dst)]

        return tasks

    def generate_steps(self):

        if self.action == 'delete':
            # Delete
            self.steps.append(DeleteTask(self.source_concept))

        elif self.action == 'rename':
            src = self.source_concept
            dst = self.target_concepts[0]

            # Rename
            if src.term != dst.term:
                for step in self.generate_replace_tasks(src, dst):
                    self.steps.append(step)

                # Note: Update source concept before next step (move)
                dst_copy = copy(dst)
                dst_copy.tag = src.tag
                src = dst_copy

            # Move
            if src.tag != dst.tag:
                self.steps.append(MoveTask(src, dst.tag))

            # Add
            if len(self.target_concepts) > 1:
                self.steps.append(AddTask(self.target_concepts[1]))

    def update_record(self, bib):

        for step in self.steps:
            step.run(bib.marc_record)

        if bib.cz_link is not None:
            log.warning(dedent(
                '''\
                Encountered a Community Zone record. Updating such records through the API will
                currently cause them to be de-linked from CZ, which is probably not what you want.
                Until Ex Libris fixes this, you're best off editing the record manually in Alma.
                '''))

            if not self.interactive or yesno('Do you want to update the record and break CZ linkage?', default='no'):
                log.warning(' -> Skipping this record. You should update it manually in Alma!')
                return

            log.warning(' -> Updating the record. The CZ connection will be lost!')

        try:
            bib.save(self.show_diffs, self.dry_run)
        except HTTPError:
            msg = '*** Failed to save record %s --- Please try to edit the record manually in Alma ***'
            log.error(msg, bib.mms_id)

    def authorize(self):
        if self.action == 'delete':
            return
        self.source_concept.authorize(self.skosmos)
        self.target_concepts[0].authorize(self.skosmos)
        if self.target_concepts[0].sf['0'] is None:
            # Use the source term identifier (if we just moved a concept)
            self.target_concepts[0].sf['0'] = self.source_concept.sf['0']
        if self.target_concepts[0].sf['0'] is None:
            log.warning('Neither the source term nor the (first) target term could be authorized in Skosmos.')
        for target_concept in self.target_concepts[1:]:
            target_concept.authorize(self.skosmos)
            if target_concept.sf['0'] is None:
                log.warning('The target term "%s" could not be authorized in Skosmos.', target_concept)

    def start(self):

        if self.alma.name is not None:
            log.info('Alma environment: %s', self.alma.name)

        for n, step in enumerate(self.steps):
            log.info('Step %d of %d: %s', n + 1, len(self.steps), step)

        if self.dry_run:
            log.info('Dry run: No catalog records will be touched!')

        # ------------------------------------------------------------------------------------
        # Del 1: Søk mot SRU for å finne over alle bibliografiske poster med emneordet.
        # Vi må filtrere resultatlista i etterkant fordi
        #  - vi mangler en egen indeks for Realfagstermer, så vi må søke mot `alma.subjects`
        #  - søket er ikke presist, så f.eks. "Monstre" vil gi treff i "Mønstre"
        #
        # I fremtiden, når vi får $0 på alle poster, kan vi bruke indeksen `alma.authority_id`
        # i stedet.

        valid_records = []
        pbar = None
        cql_query = 'alma.subjects=="%s" AND alma.authority_vocabulary = "%s"' % (self.source_concept.term,
                                                                                  self.vocabulary.marc_code)

        try:
            for marc_record in self.sru.search(cql_query):
                if pbar is None and self.show_progress and self.sru.num_records > 50:
                    pbar = tqdm(total=self.sru.num_records, desc='Filtering SRU results')

                log.debug('Checking record %s', marc_record.id())

                for step in self.steps:
                    if step.match(marc_record):
                        valid_records.append(marc_record.id())
                        break

                if pbar is not None:
                    pbar.update()
            if pbar is not None:
                pbar.close()
        except TooManyResults:
            log.error('More than 10,000 results would have to be checked, but the Alma SRU service does ' +
                      'not allow us to retrieve more than 10,000 results. Annoying? Go vote for this:\n' +
                      'http://ideas.exlibrisgroup.com/forums/308173-alma/suggestions/' +
                      '18737083-sru-srw-increase-the-10-000-record-retrieval-limi')
            return []

        if len(valid_records) == 0:
            log.info('No matching catalog records found')
            return []
        else:
            log.info('%d catalog records will be updated', len(valid_records))

            if self.interactive and not yesno('Continue?', default='yes'):
                log.info('Bye')
                return []

        # ------------------------------------------------------------------------------------
        # Del 2: Nå har vi en liste over MMS-IDer for bibliografiske poster vi vil endre.
        # Vi går gjennom dem én for én, henter ut posten med Bib-apiet, endrer og poster tilbake.

        for n, mms_id in enumerate(valid_records):
            log.info(' %3d/%d: %s', n + 1, len(valid_records), mms_id)
            bib = self.alma.bibs(mms_id)
            self.update_record(bib)

        n_posts = '{:d} {}'.format(len(valid_records), 'record' if len(valid_records) == 1 else 'records')

        subject = None

        if self.action == 'delete':
            args = (self.source_concept.tag, self.source_concept.term, n_posts)
            subject = 'Deleted {} "{}" in {}'.format(*args)

        elif self.action == 'rename':
            if len(self.target_concepts) == 2:
                args = (self.source_concept.tag, self.source_concept.term,
                        self.target_concepts[0].tag, self.target_concepts[0].term,
                        self.target_concepts[1].tag, self.target_concepts[1].term,
                        n_posts)
                subject = 'Changed {} "{}" to {} "{}" + {} "{}" in {}'.format(*args)

            elif len(self.target_concepts) == 1:
                if self.source_concept.term == self.target_concepts[0].term:
                    args = (self.source_concept.tag, self.source_concept.term,
                            self.target_concepts[0].tag, n_posts)
                    subject = 'Moved {} "{}" to {} in {}'.format(*args)
                else:
                    args = (self.source_concept.tag, self.source_concept.term,
                            self.target_concepts[0].tag, self.target_concepts[0].term,
                            n_posts)
                    subject = 'Changed {} "{}" to {} "{}" in {}'.format(*args)

        body = log_capture_string.getvalue()

        if subject is not None:
            log.info(subject)
            if self.mailer is not None and not self.dry_run:
                self.mailer.send(subject, body)

        return valid_records

{'fraction_non_alphanumeric': 0.11920155993431855, 'fraction_numerical': 0.008210180623973728, 'mean_word_length': 1.832291818049702, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6146303', 'n_tokens_mistral': 5938, 'n_tokens_neox': 5508, 'n_words': 1490}