<commit_before><commit_msg>`createstatsindex` command<commit_after>import requests
import json
from django.core.management.base import BaseCommand
from django.conf import settings

ES_INDEX_PREFIX = "_snoopstats-"
ES_MAPPINGS = {
    'task': {
        'properties': {
            'collection': {'type': 'keyword'},
            'func': {'type': 'keyword'},
            'args': {'type': 'keyword'},
            'status': {'type': 'keyword'},
            'date_created': {'type': 'date', 'format': 'date_time'},
            'date_modified': {'type': 'date', 'format': 'date_time'},
            'date_started': {'type': 'date', 'format': 'date_time'},
            'date_finished': {'type': 'date', 'format': 'date_time'},
            'duration': {'type': 'float'},
        },
    },
    'blob': {
        'properties': {
            'collection': {'type': 'keyword'},
            'mime_type': {'type': 'keyword'},
            'mime_encoding': {'type': 'keyword'},
            'date_created': {'type': 'date', 'format': 'date_time'},
            'date_modified': {'type': 'date', 'format': 'date_time'},
        },
    },
}

class Command(BaseCommand):

    help = "Set mappings for the stats indexes"

    def handle(self, name, url, index, public, **kwargs):
        for document_type in ['task', 'blob']:
            index = ES_INDEX_PREFIX + document_type
            url = f'{settings.HOOVER_ELASTICSEARCH_URL}/{index}'

            config = {'mappings': {document_type: ES_MAPPINGS[document_type]}}
            put_resp = requests.put(
                url,
                data=json.dumps(config),
                headers={'Content-Type': 'application/json'},
            )
            print('%s Elasticsearch PUT: %r', document_type, put_resp)
            print('%s Elasticsearch PUT: %r', document_type, put_resp.text)
{'fraction_non_alphanumeric': 0.17517847336628226, 'fraction_numerical': 0.0, 'mean_word_length': 2.4904214559386975, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2398451', 'n_tokens_mistral': 536, 'n_tokens_neox': 524, 'n_words': 122}