<commit_before># This file is part of the Indico plugins.
# Copyright (C) 2002 - 2021 CERN
#
# The Indico plugins are free software; you can redistribute
# them and/or modify them under the terms of the MIT License;
# see the LICENSE file for more details.

import re

from indico.core.db import db
from indico.util.console import verbose_iterator
from indico.util.string import strip_control_chars

from indico_livesync.simplify import SimpleChange, process_records


class Uploader:
    """Handles batch data upload to a remote service."""

    def __init__(self, backend, verbose=False, from_cli=False):
        self.backend = backend
        self.verbose = verbose
        self.from_cli = from_cli
        self.logger = backend.plugin.logger

    def run(self, records):
        """Runs the batch upload

        :param records: an iterable containing queue entries
        """
        self_name = type(self).__name__
        simplified = process_records(records).items()
        try:
            self.logger.info(f'{self_name} uploading %d changes from %d records', len(simplified), len(records))
            self.upload_records(simplified)
        except Exception:
            self.logger.exception(f'{self_name} failed')
            if self.from_cli:
                raise
            return
        self.processed_records(records)
        self.logger.info(f'{self_name} finished (%d total changes from %d records)', len(simplified), len(records))

    def run_initial(self, records, total):
        """Runs the initial batch upload

        :param records: an iterable containing records
        :param total: the total of records to be exported
        :return: True if everything was successful, False if not
        """
        records = verbose_iterator(
            ((rec, SimpleChange.created) for rec in records),
            total,
            lambda entry: entry[0].id,
            lambda entry: re.sub(r'\s+', ' ', strip_control_chars(getattr(entry[0], 'title', ''))),
            print_total_time=True
        )
        return self.upload_records(records, initial=True)

    def upload_records(self, records, initial=False):
        """Executed to upload records.

        :param records: an iterator of records to upload (or queue entries)
        :param initial: whether the upload is part of an initial export
        :return: True if everything was successful, False if not
        """
        raise NotImplementedError  # pragma: no cover

    def processed_records(self, records):<commit_msg>LiveSync: Show progress when running queue from CLI<commit_after># This file is part of the Indico plugins.
# Copyright (C) 2002 - 2021 CERN
#
# The Indico plugins are free software; you can redistribute
# them and/or modify them under the terms of the MIT License;
# see the LICENSE file for more details.

import re

from indico.core.db import db
from indico.util.console import verbose_iterator
from indico.util.string import strip_control_chars

from indico_livesync.simplify import SimpleChange, process_records


class Uploader:
    """Handles batch data upload to a remote service."""

    def __init__(self, backend, verbose=False, from_cli=False):
        self.backend = backend
        self.verbose = verbose
        self.from_cli = from_cli
        self.logger = backend.plugin.logger

    def run(self, records):
        """Runs the batch upload

        :param records: an iterable containing queue entries
        """
        self_name = type(self).__name__
        simplified = process_records(records).items()
        total = len(simplified)
        if self.from_cli:
            simplified = self._make_verbose(simplified, total)
        try:
            self.logger.info(f'{self_name} uploading %d changes from %d records', total, len(records))
            self.upload_records(simplified)
        except Exception:
            self.logger.exception(f'{self_name} failed')
            if self.from_cli:
                raise
            return
        self.processed_records(records)
        self.logger.info(f'{self_name} finished (%d total changes from %d records)', total, len(records))

    def run_initial(self, records, total):
        """Runs the initial batch upload

        :param records: an iterable containing records
        :param total: the total of records to be exported
        :return: True if everything was successful, False if not
        """
        records = ((rec, SimpleChange.created) for rec in records)
        records = self._make_verbose(records, total)
        return self.upload_records(records, initial=True)

    def _make_verbose(self, iterator, total):
        return verbose_iterator(
            iterator,
            total,
            lambda entry: entry[0].id,
            lambda entry: re.sub(r'\s+', ' ', strip_control_chars(getattr(entry[0], 'title', ''))),
            print_total_time=True
        )

    def upload_records(self, records, initial=False):
        """Executed to upload records.

        :param records: an iterator of records to upload (or queue entries)
        :param initial: whether the upload is part of an initial export
        :return: True if everything was successful, False if not
        """
        raise NotImplementedError  # pragma: no cover

    def processed_records(self, records):
{'fraction_non_alphanumeric': 0.0854878278920551, 'fraction_numerical': 0.0037742970371768257, 'mean_word_length': 2.6985345429169576, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5508712', 'n_tokens_mistral': 1441, 'n_tokens_neox': 1380, 'n_words': 486}