<commit_before>from datetime import datetime
import logging
import warnings
from raw.models import RawScheduleMember
from raw.processors.base import BaseProcessor, file_wrapper


logger = logging.getLogger('legcowatch')


class ScheduleMemberProcessor(BaseProcessor):
    """
    Class that handles the loading of Library Agenda scraped items into the RawCouncilAgenda table
    """
    def process(self, *args, **kwargs):
        logger.info("Processing file {}".format(self.items_file_path))
        counter = 0
        for item in file_wrapper(self.items_file_path):
            counter += 1
            self._process_item(item)
        logger.info("{} items processed, {} created, {} updated".format(counter, self._count_created, self._count_updated))

    def _process_item(self, item):
        uid = self._generate_uid(item)
        obj = self._get_object(uid)
        if obj is None:
            logger.warn(u'Could not process member item: {}'.format(item))
            return
        obj.last_parsed = datetime.now()
        if self.job is not None:
            obj.last_crawled = self.job.completed

        fields = ['last_name_c', 'first_name_c', 'last_name_e', 'first_name_e', 'english_name']
        for f in fields:
            setattr(obj, f, item.get(f, None))

        obj.save()

    def _get_object(self, uid):
        try:
            obj = RawScheduleMember.objects.get(uid=uid)
            self._count_updated += 1
        except RawScheduleMember.DoesNotExist:
            obj = RawScheduleMember(uid=uid)
            self._count_created += 1
        except RawScheduleMember.MultipleObjectsReturned:
            warnings.warn("Found more than one item with raw id {}".format(uid), RuntimeWarning)
            obj = None
        return obj

    def _generate_uid(self, item):
        return 'smember-{}'.format(item['id'])<commit_msg>Refactor of processors for Schedule api<commit_after>from datetime import datetime
import logging
import warnings

from raw.models import RawScheduleMember
from raw.processors.base import BaseProcessor, file_wrapper


logger = logging.getLogger('legcowatch')


class BaseScheduleProcessor(BaseProcessor):
    # Doing some refactoring, but don't want to affect other processors
    model = None

    def process(self, *args, **kwargs):
        logger.info("Processing file {}".format(self.items_file_path))
        counter = 0
        for item in file_wrapper(self.items_file_path):
            counter += 1
            self._process_item_wrapper(item)
        logger.info("{} items processed, {} created, {} updated".format(counter, self._count_created, self._count_updated))

    def _process_item(self, item, obj):
        raise NotImplementedError()

    def _process_item_wrapper(self, item):
        uid = self._generate_uid(item)
        obj = self._get_object(uid)
        if obj is None:
            logger.warn(u'Could not process member item: {}'.format(item))
            return
        obj.last_parsed = datetime.now()
        if self.job is not None:
            obj.last_crawled = self.job.completed
        self._process_item(item, obj)

    def _get_object(self, uid):
        try:
            obj = self.model.objects.get(uid=uid)
            self._count_updated += 1
        except RawScheduleMember.DoesNotExist:
            obj = self.model(uid=uid)
            self._count_created += 1
        except self.model.MultipleObjectsReturned:
            warnings.warn("Found more than one item with raw id {}".format(uid), RuntimeWarning)
            obj = None
        return obj

    def _generate_uid(self, item):
        raise NotImplementedError()


class ScheduleMemberProcessor(BaseScheduleProcessor):
    """
    Class that handles the loading of Library Agenda scraped items into the RawCouncilAgenda table
    """
    model = RawScheduleMember

    def _process_item(self, item, obj):
        fields = ['last_name_c', 'first_name_c', 'last_name_e', 'first_name_e', 'english_name']
        for f in fields:
            setattr(obj, f, item.get(f, None))
        obj.save()

    def _generate_uid(self, item):
        return 'smember-{}'.format(item['id'])
{'fraction_non_alphanumeric': 0.09884001933301112, 'fraction_numerical': 0.0019333011116481392, 'mean_word_length': 2.7457013574660634, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1439391', 'n_tokens_mistral': 1175, 'n_tokens_neox': 1147, 'n_words': 307}