<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Load data into processed CAL-ACCESS models, archive processed files and ZIP.
"""
import os
from django.conf import settings
from django.core.files import File
from django.utils.timezone import now
from django.core.management import call_command
from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile
from calaccess_processed.management.commands import CalAccessCommand
from calaccess_processed.models import ProcessedDataZip


class Command(CalAccessCommand):
    """
    Load data into processed CAL-ACCESS models, archive processed files and ZIP.
    """
    help = 'Load data into processed CAL-ACCESS models, archive processed files and ZIP.'

    def add_arguments(self, parser):
        """
        Adds custom arguments specific to this command.
        """
        parser.add_argument(
            "--force-restart",
            "--restart",
            action="store_true",
            dest="restart",
            default=False,
            help="Force re-start (overrides auto-resume)."
        )

    def handle(self, *args, **options):
        """
        Make it happen.
        """
        # Set options
        super(Command, self).handle(*args, **options)
        self.force_restart = options.get("restart")

        # Get or create the logger record
        self.processed_version, created = self.get_or_create_processed_version()

        # If the version is already fully processed and we're not forcing a do-over
        # then just quit out now
        if self.processed_version.update_completed and not self.force_restart:
            msg_tmp = 'Processing completed at %s.'
            self.success(
                msg_tmp % self.processed_version.process_finish_datetime.ctime()
            )
            return False

        # Otherwise proceed with the standard routine
        # Firt, start the clock if created (or restart if forcing restart)
        if created or self.force_restart:
            self.processed_version.process_start_datetime = now()
            # also reset finish time if forcing re-start
            if self.force_restart:
                self.processed_version.process_finish_datetime = None
            self.processed_version.save()
            if self.verbosity > 2:
                self.log('Flushing local copies of processed data files.')
            self.flush_data_files()

        # then load
        # self.load()

        #then verify
        call_command(
            'verifycalaccessprocesseddata',
            verbosity=self.verbosity,
            no_color=self.no_color,
        )

        # then zip
        flat_zip_path = self.zip('flat')
        relational_zip_path = self.zip('relational')

        # then archive
        if getattr(settings, 'CALACCESS_STORE_ARCHIVE', False):
            self.archive_zip(flat_zip_path)
            self.archive_zip(relational_zip_path)
<commit_msg>flake8<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Load data into processed CAL-ACCESS models, archive processed files and ZIP.
"""
import os
from django.conf import settings
from django.core.files import File
from django.utils.timezone import now
from django.core.management import call_command
from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile
from calaccess_processed.management.commands import CalAccessCommand
from calaccess_processed.models import ProcessedDataZip


class Command(CalAccessCommand):
    """
    Load data into processed CAL-ACCESS models, archive processed files and ZIP.
    """
    help = 'Load data into processed CAL-ACCESS models, archive processed files and ZIP.'

    def add_arguments(self, parser):
        """
        Adds custom arguments specific to this command.
        """
        parser.add_argument(
            "--force-restart",
            "--restart",
            action="store_true",
            dest="restart",
            default=False,
            help="Force re-start (overrides auto-resume)."
        )

    def handle(self, *args, **options):
        """
        Make it happen.
        """
        # Set options
        super(Command, self).handle(*args, **options)
        self.force_restart = options.get("restart")

        # Get or create the logger record
        self.processed_version, created = self.get_or_create_processed_version()

        # If the version is already fully processed and we're not forcing a do-over
        # then just quit out now
        if self.processed_version.update_completed and not self.force_restart:
            msg_tmp = 'Processing completed at %s.'
            self.success(
                msg_tmp % self.processed_version.process_finish_datetime.ctime()
            )
            return False

        # Otherwise proceed with the standard routine
        # Firt, start the clock if created (or restart if forcing restart)
        if created or self.force_restart:
            self.processed_version.process_start_datetime = now()
            # also reset finish time if forcing re-start
            if self.force_restart:
                self.processed_version.process_finish_datetime = None
            self.processed_version.save()
            if self.verbosity > 2:
                self.log('Flushing local copies of processed data files.')
            self.flush_data_files()

        # then load
        # self.load()

        # then verify
        call_command(
            'verifycalaccessprocesseddata',
            verbosity=self.verbosity,
            no_color=self.no_color,
        )

        # then zip
        flat_zip_path = self.zip('flat')
        relational_zip_path = self.zip('relational')

        # then archive
        if getattr(settings, 'CALACCESS_STORE_ARCHIVE', False):
            self.archive_zip(flat_zip_path)
            self.archive_zip(relational_zip_path)

{'fraction_non_alphanumeric': 0.0823327615780446, 'fraction_numerical': 0.0008576329331046312, 'mean_word_length': 2.279527559055118, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1217467', 'n_tokens_mistral': 1565, 'n_tokens_neox': 1506, 'n_words': 434}