<commit_before>import argparse
import json
import os

os.environ['DJANGO_SETTINGS_MODULE'] = 'thresher_backend.settings'
from django.conf import settings

from django.core.management import call_command
from django.core.management.color import no_style
from django.core.management.sql import sql_delete
from django.db import connections, DEFAULT_DB_ALIAS, models
from django.db.utils import IntegrityError
from django.core.exceptions import ValidationError

from data.parse_document import parse_document
from data.parse_schema import parse_schema
from parse_schema import TopicsSchemaParser
from thresher.models import Article, Topic, HighlightGroup, ArticleHighlight
ANALYSIS_TYPES = {}
HIGH_ID = 20000

def load_schema(schema):
    schema_name = schema['title']
    schema_parent = schema['parent']
    if schema_parent:
        parent = Topic.objects.get(name=schema_parent)
    else:
        parent = None
    schema_obj = Topic(
        parent=parent,
        name=schema_name,
        instructions=schema['instructions'],
        glossary=json.dumps(schema['glossary'])
    )
    try:
        schema_obj.save()
    except ValidationError:
        # we've already loaded this schema, pull it into memory.
        print "Schema already exists. It will be overwritten"
        curr_schema_obj = Topic.objects.get(name=schema_name)
        # We can't just delete the object because this will delete all TUAs associated with it.
        # Instead, we update the Analysis Type and delete all the topics associated with it.
        # When the id is set, django automatically knows to update instead of creating a new entry.
        schema_obj.id = curr_schema_obj.id
        # Save the updated object
        schema_obj.save()
        # delete all topics associated with this Analysis Type
        # This will CASCADE DELETE all questions and answers as well
        Topic.objects.filter(parent=schema_obj).delete()

    ANALYSIS_TYPES[schema_name] = schema_obj
    print "loading schema:", schema_name

    # Load the topics, questions and answers of the schema
    schema_parser = TopicsSchemaParser(topic_obj=schema_obj, 
                                       schema=schema['topics'],
                                       dependencies=schema['dependencies'])
    schema_parser.load_topics()

def load_article(article):
    new_id = int(article['metadata']['article_id'])

    try: # Catch duplicate article ids and assign new ids.
        existing_article = Article.objects.get(article_id=new_id)
        if article['text'] != existing_article.text:
            max_id = Article.objects.all().order_by('-article_id')[0].article_id
            new_id = max_id + 1 if max_id >= HIGH_ID else HIGH_ID
            print "NEW ID!", new_id
        else:
            # we've already loaded this article, so don't process its TUAs.
            return

    except Article.DoesNotExist: # Not a duplicate.
        pass

    article_obj = Article(
        article_id=new_id,
        text=article['text'],
        date_published=article['metadata']['date_published'],
        city_published=article['metadata']['city'],
        state_published=article['metadata']['state'],
        periodical=article['metadata']['periodical'],
        periodical_code=int(article['metadata']['periodical_code']),
        parse_version=article['metadata']['version'],
        annotators=json.dumps(article['metadata']['annotators']),
    )
    article_obj.save()

    for tua_type, tuas in article['tuas'].iteritems():
        offsets = []
        try:
            topic = Topic.objects.filter(name=tua_type)[0]
            #analysis_type = (ANALYSIS_TYPES.get(tua_type) or
            #                 Topic.objects.get(name=tua_type))
        except IndexError:
            # No analysis type loaded--create a dummy type.
            topic = Topic.objects.create(
                name=tua_type,
                instructions='',
                glossary='',
            )
            ANALYSIS_TYPES[tua_type] = topic
            print("made a dummy topic")
#           raise ValueError("No TUA type '" + tua_type +
#                            "' registered. Have you loaded the schemas?")
        for tua_id, offset_list in tuas.iteritems():
            offsets.extend(offset_list)

        highlight = HighlightGroup.objects.create(offsets=json.dumps(offsets))
        try:
            ArticleHighlight.objects.create(topic=topic,
                                            highlight=highlight, 
                                            article=article_obj)
        except ValidationError as e:
            print 'error on article #', new_id, 'tua #', tua_id, 'of', tua_type
            print e

    print 'loading article...'

def load_schema_dir(dirpath):
    for schema_file in os.listdir(dirpath):
        if os.path.splitext(schema_file)[1] != '.txt':
            continue
        load_schema(parse_schema(os.path.join(dirpath, schema_file)))

def load_article_dir(dirpath):
    for article_file in os.listdir(dirpath):
        if os.path.splitext(article_file)[1] != '.txt':
            continue
        load_article(parse_document(os.path.join(dirpath, article_file)))

def load_args():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '-s', '--schema-dir',
        help='The directory holding raw schema files for the TUA types')<commit_msg>Lexically sort schema file names to satisfy order dependencies. (#11)<commit_after>import argparse
import json
import os
import fnmatch

os.environ['DJANGO_SETTINGS_MODULE'] = 'thresher_backend.settings'
from django.conf import settings

from django.core.management import call_command
from django.core.management.color import no_style
from django.core.management.sql import sql_delete
from django.db import connections, DEFAULT_DB_ALIAS, models
from django.db.utils import IntegrityError
from django.core.exceptions import ValidationError

from data.parse_document import parse_document
from data.parse_schema import parse_schema
from parse_schema import TopicsSchemaParser
from thresher.models import Article, Topic, HighlightGroup, ArticleHighlight
ANALYSIS_TYPES = {}
HIGH_ID = 20000

def load_schema(schema):
    schema_name = schema['title']
    schema_parent = schema['parent']
    if schema_parent:
        parent = Topic.objects.get(name=schema_parent)
    else:
        parent = None
    schema_obj = Topic(
        parent=parent,
        name=schema_name,
        instructions=schema['instructions'],
        glossary=json.dumps(schema['glossary'])
    )
    try:
        schema_obj.save()
    except ValidationError:
        # we've already loaded this schema, pull it into memory.
        print "Schema already exists. It will be overwritten"
        curr_schema_obj = Topic.objects.get(name=schema_name)
        # We can't just delete the object because this will delete all TUAs associated with it.
        # Instead, we update the Analysis Type and delete all the topics associated with it.
        # When the id is set, django automatically knows to update instead of creating a new entry.
        schema_obj.id = curr_schema_obj.id
        # Save the updated object
        schema_obj.save()
        # delete all topics associated with this Analysis Type
        # This will CASCADE DELETE all questions and answers as well
        Topic.objects.filter(parent=schema_obj).delete()

    ANALYSIS_TYPES[schema_name] = schema_obj
    print "loading schema:", schema_name

    # Load the topics, questions and answers of the schema
    schema_parser = TopicsSchemaParser(topic_obj=schema_obj, 
                                       schema=schema['topics'],
                                       dependencies=schema['dependencies'])
    schema_parser.load_topics()

def load_article(article):
    new_id = int(article['metadata']['article_id'])

    try: # Catch duplicate article ids and assign new ids.
        existing_article = Article.objects.get(article_id=new_id)
        if article['text'] != existing_article.text:
            max_id = Article.objects.all().order_by('-article_id')[0].article_id
            new_id = max_id + 1 if max_id >= HIGH_ID else HIGH_ID
            print "NEW ID!", new_id
        else:
            # we've already loaded this article, so don't process its TUAs.
            return

    except Article.DoesNotExist: # Not a duplicate.
        pass

    article_obj = Article(
        article_id=new_id,
        text=article['text'],
        date_published=article['metadata']['date_published'],
        city_published=article['metadata']['city'],
        state_published=article['metadata']['state'],
        periodical=article['metadata']['periodical'],
        periodical_code=int(article['metadata']['periodical_code']),
        parse_version=article['metadata']['version'],
        annotators=json.dumps(article['metadata']['annotators']),
    )
    article_obj.save()

    for tua_type, tuas in article['tuas'].iteritems():
        offsets = []
        try:
            topic = Topic.objects.filter(name=tua_type)[0]
            #analysis_type = (ANALYSIS_TYPES.get(tua_type) or
            #                 Topic.objects.get(name=tua_type))
        except IndexError:
            # No analysis type loaded--create a dummy type.
            topic = Topic.objects.create(
                name=tua_type,
                instructions='',
                glossary='',
            )
            ANALYSIS_TYPES[tua_type] = topic
            print("made a dummy topic")
#           raise ValueError("No TUA type '" + tua_type +
#                            "' registered. Have you loaded the schemas?")
        for tua_id, offset_list in tuas.iteritems():
            offsets.extend(offset_list)

        highlight = HighlightGroup.objects.create(offsets=json.dumps(offsets))
        try:
            ArticleHighlight.objects.create(topic=topic,
                                            highlight=highlight, 
                                            article=article_obj)
        except ValidationError as e:
            print 'error on article #', new_id, 'tua #', tua_id, 'of', tua_type
            print e

    print 'loading article...'

def load_schema_dir(dirpath):
    schema_files = sorted(fnmatch.filter(os.listdir(dirpath), '*.txt'))
    for schema_file in schema_files:
        load_schema(parse_schema(os.path.join(dirpath, schema_file)))

def load_article_dir(dirpath):
    for article_file in os.listdir(dirpath):
        if os.path.splitext(article_file)[1] != '.txt':
            continue
        load_article(parse_document(os.path.join(dirpath, article_file)))

def load_args():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '-s', '--schema-dir',
        help='The directory holding raw schema files for the TUA types')
{'fraction_non_alphanumeric': 0.09404010397326401, 'fraction_numerical': 0.0019494987003341997, 'mean_word_length': 2.6260518343991923, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1244102', 'n_tokens_mistral': 3000, 'n_tokens_neox': 2937, 'n_words': 749}