<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division

from optparse import make_option

import os
import json
from functools import partial

from django.conf import settings
from django.db.transaction import commit_on_success

from treemap import SPECIES
from treemap.models import User, Species, InstanceUser
from treemap.management.util import InstanceDataCommand

from otm1_migrator.models import OTM1UserRelic, OTM1ModelRelic
from otm1_migrator.migration_rules import MIGRATION_RULES
from otm1_migrator.data_util import hash_to_model, MigrationException


def find_user_to_save_with(model):
    model_name = model.__class__.__name__.lower()

    user_field_to_try = (MIGRATION_RULES[model_name]
                         .get('dependencies', {})
                         .get('user', None))

    if user_field_to_try:
        potential_user_id = getattr(model, user_field_to_try, None)
    else:
        potential_user_id = None

    try:
        user = User.objects.get(pk=potential_user_id)
    except User.DoesNotExist:
        user = User.system_user()

    return user


def save_model_with_user(model, instance):
    user = find_user_to_save_with(model)
    model.save_with_user_without_verifying_authorization(user)
    return model


def overwrite_old_pks(model_hash, model_name, dependency_ids):

    dependencies = (MIGRATION_RULES
                    .get(model_name, {})
                    .get('dependencies', {})
                    .items())

    # rewrite the fixture so that otm1 pks are replaced by
    # their corresponding otm2 pks
    if dependencies:
        for name, field in dependencies:
            old_id = model_hash['fields'][field]
            if old_id:
                old_id_to_new_id = dependency_ids[name]
                try:
                    new_id = old_id_to_new_id[old_id]
                except KeyError:
                    raise MigrationException("Dependency not found. "
                                             "Have you imported %s yet?"
                                             % name)
                model_hash['fields'][field] = new_id


@commit_on_success
def save_species(model_hash, instance):
    # Does this species already exist?
    genus, species, cultivar, other = [
        model_hash['fields'].get(thing, '')
        for thing in ['genus', 'species', 'cultivar', 'other_part_of_name']]

    existingspecies = Species.objects.filter(genus=genus,
                                             species=species,
                                             cultivar=cultivar,
                                             other=other)

    relicinfo = {
        'instance': instance,
        'otm1_model_id': model_hash['pk'],
        'otm2_model_name': 'species'
    }

    relics = OTM1ModelRelic.objects.filter(**relicinfo)

    if len(relics) == 0:
        relic = OTM1ModelRelic(**relicinfo)
    else:
        relic = relics[0]

    if len(existingspecies) == 0:
        for sp in SPECIES:
            if ((sp['genus'] == genus and
                 sp['species'] == species and
                 sp['cultivar'] == cultivar and
                 sp['other'] == other)):
                fields = model_hash['fields']
                fields['otm_code'] = sp['otm_code']

                break

        model = hash_to_model(MIGRATION_RULES,
                              'species', model_hash,
                              instance)

        if not model.common_name:
            model.common_name = ''

        model = save_model_with_user(model, instance)

        relic.otm2_model_id = model.pk
    else:
        model = None
        relic.otm2_model_id = existingspecies[0].pk

    relic.save()

    return model


@commit_on_success<commit_msg>Change field generation logic in save_species<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division

from optparse import make_option

import os
import json
from functools import partial

from django.conf import settings
from django.db.transaction import commit_on_success

from treemap import SPECIES
from treemap.models import User, Species, InstanceUser
from treemap.management.util import InstanceDataCommand

from otm1_migrator.models import OTM1UserRelic, OTM1ModelRelic
from otm1_migrator.migration_rules import MIGRATION_RULES
from otm1_migrator.data_util import hash_to_model, MigrationException


def find_user_to_save_with(model):
    model_name = model.__class__.__name__.lower()

    user_field_to_try = (MIGRATION_RULES[model_name]
                         .get('dependencies', {})
                         .get('user', None))

    if user_field_to_try:
        potential_user_id = getattr(model, user_field_to_try, None)
    else:
        potential_user_id = None

    try:
        user = User.objects.get(pk=potential_user_id)
    except User.DoesNotExist:
        user = User.system_user()

    return user


def save_model_with_user(model, instance):
    user = find_user_to_save_with(model)
    model.save_with_user_without_verifying_authorization(user)
    return model


def overwrite_old_pks(model_hash, model_name, dependency_ids):

    dependencies = (MIGRATION_RULES
                    .get(model_name, {})
                    .get('dependencies', {})
                    .items())

    # rewrite the fixture so that otm1 pks are replaced by
    # their corresponding otm2 pks
    if dependencies:
        for name, field in dependencies:
            old_id = model_hash['fields'][field]
            if old_id:
                old_id_to_new_id = dependency_ids[name]
                try:
                    new_id = old_id_to_new_id[old_id]
                except KeyError:
                    raise MigrationException("Dependency not found. "
                                             "Have you imported %s yet?"
                                             % name)
                model_hash['fields'][field] = new_id


@commit_on_success
def save_species(model_hash, instance):
    # Does this species already exist?
    genus, species, cultivar, other = [
        model_hash['fields'].get(thing, '')
        for thing in ['genus', 'species', 'cultivar', 'other_part_of_name']]

    existingspecies = Species.objects.filter(genus=genus,
                                             species=species,
                                             cultivar=cultivar,
                                             other=other)

    relicinfo = {
        'instance': instance,
        'otm1_model_id': model_hash['pk'],
        'otm2_model_name': 'species'
    }

    relics = OTM1ModelRelic.objects.filter(**relicinfo)

    if len(relics) == 0:
        relic = OTM1ModelRelic(**relicinfo)
    else:
        relic = relics[0]

    if len(existingspecies) == 0:

        model = hash_to_model(MIGRATION_RULES,
                              'species', model_hash,
                              instance)

        for sp in SPECIES:
            if ((sp['genus'] == genus and
                 sp['species'] == species and
                 sp['cultivar'] == cultivar and
                 sp['other'] == other)):
                model.otm_code = sp['otm_code']
                break

        # this field can be null in otm1, but not otm2
        if not model.common_name:
            model.common_name = ''

        model = save_model_with_user(model, instance)

        relic.otm2_model_id = model.pk
    else:
        model = None
        relic.otm2_model_id = existingspecies[0].pk

    relic.save()

    return model


@commit_on_success
{'fraction_non_alphanumeric': 0.08184543203440636, 'fraction_numerical': 0.004952430600808028, 'mean_word_length': 2.0320031608060054, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4264358', 'n_tokens_mistral': 2281, 'n_tokens_neox': 2182, 'n_words': 494}