<commit_before>import simplejson as json
from pathlib import Path
from django.core.management.base import BaseCommand
from django.conf import settings
from elasticsearch import Elasticsearch
from maldini.models import Document, Generation, Failure, Cache
from maldini.prepare import extract

es = Elasticsearch(settings.ELASTICSEARCH_URL)

class Command(BaseCommand):

    help = "Push files to ES"

    def handle(self, **options):

        offset = 0
        for doc in Document.objects.exclude(generation__n=1).order_by('id').all().iterator():
            print(doc.id, doc.path)

            try:
                data = extract(doc)

            except KeyboardInterrupt:
                raise

            except:
                Failure.objects.create(document=doc)
                print 'ERROR'

            else:
                Cache.objects.create(document=doc, data=json.dumps(data))

            Generation.objects.create(document=doc, n=1)

            #es.index(
            #    index='hoover-6',
            #    doc_type='doc',
            #    id=doc.id,
            #    body=data,
            #)
            #doc.push = False
            #doc.save()<commit_msg>encode json in the try/catch block<commit_after>import simplejson as json
from pathlib import Path
from django.core.management.base import BaseCommand
from django.conf import settings
from elasticsearch import Elasticsearch
from maldini.models import Document, Generation, Failure, Cache
from maldini.prepare import extract

es = Elasticsearch(settings.ELASTICSEARCH_URL)

class Command(BaseCommand):

    help = "Push files to ES"

    def handle(self, **options):

        offset = 0
        for doc in Document.objects.exclude(generation__n=1).order_by('id').all().iterator():
            print(doc.id, doc.path)

            try:
                data = extract(doc)
                data_json = json.dumps(data)

            except KeyboardInterrupt:
                raise

            except:
                Failure.objects.create(document=doc)
                print 'ERROR'

            else:
                Cache.objects.create(document=doc, data=data_json)

            Generation.objects.create(document=doc, n=1)

            #es.index(
            #    index='hoover-6',
            #    doc_type='doc',
            #    id=doc.id,
            #    body=data,
            #)
            #doc.push = False
            #doc.save()
{'fraction_non_alphanumeric': 0.09334985543163982, 'fraction_numerical': 0.003304419661296985, 'mean_word_length': 1.9040767386091126, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1869217', 'n_tokens_mistral': 675, 'n_tokens_neox': 639, 'n_words': 154}