<commit_before>from django.core.management.base import BaseCommand
import os
import json
import time
import sys
from facilities.models import Facility, Variable, CalculatedVariable, \
    KeyRename, FacilityRecord
from facilities.facility_builder import FacilityBuilder
from utils.csv_reader import CsvReader
from django.conf import settings

from facility_views.models import *

class Command(BaseCommand):
    help = "Load the table defs from the csvs."

    def handle(self, *args, **kwargs):
        table_types = [
            ("Health", "health"),
            ("Education", "education"),
            ("Water", "water")
        ]
        FacilityTable.objects.all().delete()
        TableColumn.objects.all().delete()
        ColumnCategory.objects.all().delete()
        
        subgroups = {}
        sgs = list(CsvReader(os.path.join("data","table_definitions", "subgroups.csv")).iter_dicts())
        for sg in sgs:
            subgroups[sg['slug']] = sg['name']

        for name, slug in table_types:
            curtable = FacilityTable.objects.create(name=name, slug=slug)
            csv_reader = CsvReader(os.path.join("data","table_definitions", "%s.csv" % slug))
            display_order = 0
            for input_d in csv_reader.iter_dicts():
                subs = []
                for sg in input_d['subgroups'].split(" "):
                    if sg in subgroups:
                        subs.append({
                            'name': subgroups[sg],
                            'slug': sg
                        })
                for sub in subs:
                    curtable.add_column(sub)
                d = {
                    'name': input_d['name'],
                    'slug': input_d['slug'],
                    'subgroups': input_d['subgroups'],
                    'description': input_d.pop('description', ''),
                    'clickable': input_d.pop('clickable', 'no') == 'yes',
                    'click_action': input_d.pop('click action', input_d.pop('click_action', '')),
                    'display_style': input_d.pop('display style', input_d.pop('display_style', '')),
                    'calc_action': input_d.pop('calc action', input_d.pop('calc_action', '')),
                    'iconify_png_url': input_d.pop('iconify_png_url', input_d.pop('iconify png url', '')),
                    'calc_columns': input_d.pop('calc columns', input_d.pop('calc_columns', '')),
                    'display_order': display_order
                }
                display_order += 1
                curtable.add_variable(d)
<commit_msg>debug message for table_defs that don't load<commit_after>from django.core.management.base import BaseCommand
import os
import json
import time
import sys
from facilities.models import Facility, Variable, CalculatedVariable, \
    KeyRename, FacilityRecord
from facilities.facility_builder import FacilityBuilder
from utils.csv_reader import CsvReader
from django.conf import settings

from facility_views.models import *

class Command(BaseCommand):
    help = "Load the table defs from the csvs."

    def handle(self, *args, **kwargs):
        table_types = [
            ("Health", "health"),
            ("Education", "education"),
            ("Water", "water")
        ]
        FacilityTable.objects.all().delete()
        TableColumn.objects.all().delete()
        ColumnCategory.objects.all().delete()
        
        subgroups = {}
        sgs = list(CsvReader(os.path.join("data","table_definitions", "subgroups.csv")).iter_dicts())
        for sg in sgs:
            subgroups[sg['slug']] = sg['name']

        for name, slug in table_types:
            curtable = FacilityTable.objects.create(name=name, slug=slug)
            csv_reader = CsvReader(os.path.join("data","table_definitions", "%s.csv" % slug))
            display_order = 0
            for input_d in csv_reader.iter_dicts():
                subs = []
                for sg in input_d['subgroups'].split(" "):
                    if sg in subgroups:
                        subs.append({
                            'name': subgroups[sg],
                            'slug': sg
                        })
                for sub in subs:
                    curtable.add_column(sub)
                try:
                    d = {
                        'name': input_d['name'],
                        'slug': input_d['slug'],
                        'subgroups': input_d['subgroups'],
                        'description': input_d.pop('description', ''),
                        'clickable': input_d.pop('clickable', 'no') == 'yes',
                        'click_action': input_d.pop('click action', input_d.pop('click_action', '')),
                        'display_style': input_d.pop('display style', input_d.pop('display_style', '')),
                        'calc_action': input_d.pop('calc action', input_d.pop('calc_action', '')),
                        'iconify_png_url': input_d.pop('iconify_png_url', input_d.pop('iconify png url', '')),
                        'calc_columns': input_d.pop('calc columns', input_d.pop('calc_columns', '')),
                        'display_order': display_order
                    }
                    display_order += 1
                    curtable.add_variable(d)
                except:
                    print "Error importing table definition for data: %s" % input_d

{'fraction_non_alphanumeric': 0.12504638218923933, 'fraction_numerical': 0.0007421150278293135, 'mean_word_length': 1.826953329837441, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '822301', 'n_tokens_mistral': 1496, 'n_tokens_neox': 1414, 'n_words': 307}