<commit_before># -*- coding: utf-8 -*-
import logging
from django.core.exceptions import ObjectDoesNotExist
from dplace_app.models import *
from isocode import get_isocode, get_value
from environmental import iso_from_code

MISSING_CODES = []
def add_missing_isocode(isocode):
    MISSING_CODES.append(isocode)

# OLD CODE FOR ETHNOLOGUE CLASSIFICATIONS
#---WILL NOT WORK AS DEPENDS ON LANGUAGECLASS and LANGUAGECLASSIFICATION MODELS
#def load_lang(lang_row):
    # Extract values from dictionary
    code = get_isocode(lang_row)
    if code is None:
        logging.warn("No ISO Code found, skipping")
        return None
    language_name = get_value(lang_row,('Language name','NAM'))
    #ethnologue_classification = get_value(lang_row,('Ethnologue Classification (unrevised)','Ethnologue classification (if applicable)'))
#    family_names = [
#        get_value(lang_row,('FAMILY-REVISED','FAMILY')),
#        lang_row['Class2'],
#        lang_row['Class3']
#    ]
    # ISO Code
    isocode = iso_from_code(code) # Does not create new ISO Codes
    if isocode is None:
        logging.warn(
            "No ISO Code found in database for %s, skipping language %s" % (code, language_name)
        )
        add_missing_isocode(code)
        return None

    # Language
    try:
        language = Language.objects.get(iso_code=isocode)
    except ObjectDoesNotExist:
        language = Language(name=language_name, iso_code=isocode)
        language.save()
        logging.info("Created Language: %s" % language_name)
    
    # Classes
#    classes = []
#    for i in range(3):
#        level = i + 1
#        name = family_names[i].strip()
#        if len(name) == 0:
            # empty cell
            continue
        try:
            classes.append(LanguageClass.objects.get(scheme='R',level=level,name=name))
        except ObjectDoesNotExist:
            if len(classes) > 0:
                parent = classes[-1]
            else:<commit_msg>added missing "#" in front "continue"<commit_after># -*- coding: utf-8 -*-
import logging
from django.core.exceptions import ObjectDoesNotExist
from dplace_app.models import *
from isocode import get_isocode, get_value
from environmental import iso_from_code

MISSING_CODES = []
def add_missing_isocode(isocode):
    MISSING_CODES.append(isocode)

# OLD CODE FOR ETHNOLOGUE CLASSIFICATIONS
#---WILL NOT WORK AS DEPENDS ON LANGUAGECLASS and LANGUAGECLASSIFICATION MODELS
#def load_lang(lang_row):
    # Extract values from dictionary
    code = get_isocode(lang_row)
    if code is None:
        logging.warn("No ISO Code found, skipping")
        return None
    language_name = get_value(lang_row,('Language name','NAM'))
    #ethnologue_classification = get_value(lang_row,('Ethnologue Classification (unrevised)','Ethnologue classification (if applicable)'))
#    family_names = [
#        get_value(lang_row,('FAMILY-REVISED','FAMILY')),
#        lang_row['Class2'],
#        lang_row['Class3']
#    ]
    # ISO Code
    isocode = iso_from_code(code) # Does not create new ISO Codes
    if isocode is None:
        logging.warn(
            "No ISO Code found in database for %s, skipping language %s" % (code, language_name)
        )
        add_missing_isocode(code)
        return None

    # Language
    try:
        language = Language.objects.get(iso_code=isocode)
    except ObjectDoesNotExist:
        language = Language(name=language_name, iso_code=isocode)
        language.save()
        logging.info("Created Language: %s" % language_name)
    
    # Classes
#    classes = []
#    for i in range(3):
#        level = i + 1
#        name = family_names[i].strip()
#        if len(name) == 0:
            # empty cell
            # continue
        try:
            classes.append(LanguageClass.objects.get(scheme='R',level=level,name=name))
        except ObjectDoesNotExist:
            if len(classes) > 0:
                parent = classes[-1]
            else:
{'fraction_non_alphanumeric': 0.10259014728288471, 'fraction_numerical': 0.004062976130015236, 'mean_word_length': 2.8504398826979473, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4863161', 'n_tokens_mistral': 1274, 'n_tokens_neox': 1229, 'n_words': 293}