<commit_before>import logging
import datetime
import time
import timeit

from django.core.management.base import BaseCommand, CommandError
from django.db import transaction

import snotes20.models as models
import snotes20.contenttypes as contenttypes


logger = logging.getLogger(__name__)


def update_document(doc):
    with transaction.atomic():
        try:
            doc.state.delete()
            doc.raw_state.delete()
        except:
            pass

        raw_state, state = contenttypes.get_state(doc)

        doc.raw_state = raw_state
        doc.state = state

        raw_state.save()
        state.save()
        doc.save()



class Command(BaseCommand):
    args = ''
    help = ''

    def handle(self, *args, **options):

        while True:
            start_time = datetime.datetime.now()

            today = datetime.datetime.combine(datetime.date.today(), datetime.time.min)
            docs = models.Document.objects.all().filter(edit_date__gt=today)

            for doc in docs:
                logger.debug("Updating document:" + doc.name)
                update_document(doc)

            duration = datetime.datetime.now() - start_time

            logger.debug("took {}s".format(duration.total_seconds()))

            time.sleep(1)<commit_msg>refreshdocstate: allow all (not just recent) documents to be updated via cli<commit_after>import logging
import datetime
import time
import timeit

from django.core.management.base import BaseCommand, CommandError
from django.db import transaction

import snotes20.models as models
import snotes20.contenttypes as contenttypes


logger = logging.getLogger(__name__)


def update_document(doc):
    with transaction.atomic():
        try:
            doc.state.delete()
            doc.raw_state.delete()
        except:
            pass

        raw_state, state = contenttypes.get_state(doc)

        doc.raw_state = raw_state
        doc.state = state

        raw_state.save()
        state.save()
        doc.save()


def update_documents(docs):
    start_time = datetime.datetime.now()

    for doc in docs:
        logger.debug("Updating document:" + doc.name)
        update_document(doc)

    duration = datetime.datetime.now() - start_time

    logger.debug("took {}s".format(duration.total_seconds()))


class Command(BaseCommand):
    args = ''
    help = ''

    def handle(self, *args, **options):
        mode = 'continuous'

        if len(args) == 1:
            mode = args[0]

        if mode == 'continuous':
            while True:
                today = datetime.datetime.combine(datetime.date.today(), datetime.time.min)
                docs = models.Document.objects.all().filter(edit_date__gt=today)
                update_documents(docs)
                time.sleep(1)
        elif mode == 'all':
            docs = models.Document.objects.all()
            update_documents(docs)
{'fraction_non_alphanumeric': 0.10086655112651646, 'fraction_numerical': 0.004159445407279029, 'mean_word_length': 2.324884792626728, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3426582', 'n_tokens_mistral': 826, 'n_tokens_neox': 770, 'n_words': 183}