<commit_before>"""Code for handling data from mtgjson."""

import datetime

from mtgcdb import models


def update_models(session, mtg_data, include_online_only):
    """Update database with data from mtgjson (assumes models exist).

    Arguments:
        session: Session, sqlalchemy session to read/write database.
        mtg_data: dict, data from json.load(s) on mtgjson data.
        include_online_only: process online_only sets? if False, skip.
    """
    for set_data in mtg_data.values():
        if not include_online_only and set_data.get('onlineOnly', False):
            continue

        card_set = create_set(set_data)
        session.merge(card_set)

        for card_data in set_data['cards']:
            card = create_card(card_data)
            session.merge(card)

            printing = create_printing(card_data, set_data['code'])
            session.merge(printing)


def create_set(set_data):
    """Given set data, create a CardSet."""
    card_set = models.CardSet(code=set_data['code'])
    card_set.name = set_data['name']
    card_set.block = set_data.get('block')
    card_set.release_date = datetime.datetime.strptime(
        set_data['releaseDate'], '%Y-%m-%d').date()
    card_set.type = set_data['type']<commit_msg>Optimize data loads to avoid merge implicit queries.<commit_after>"""Code for handling data from mtgjson."""

import datetime

from mtgcdb import models


def update_models(session, mtg_data, include_online_only):
    """Update database with data from mtgjson (assumes models exist).

    Arguments:
        session: Session, sqlalchemy session to read/write database.
        mtg_data: dict, data from json.load(s) on mtgjson data.
        include_online_only: process online_only sets? if False, skip.
    """
    seen_set_codes = set()
    seen_card_names = set()
    seen_printing_ids = set()
    for set_data in mtg_data.values():
        if not include_online_only and set_data.get('onlineOnly', False):
            continue

        if set_data['code'] in seen_set_codes:
            continue

        card_set = create_set(set_data)
        session.add(card_set)
        seen_set_codes.add(card_set.code)

        for card_data in set_data['cards']:
            if card_data['name'] not in seen_card_names:
                card = create_card(card_data)
                session.add(card)
                seen_card_names.add(card.name)

            if card_data['id'] not in seen_printing_ids:
                printing = create_printing(card_data, set_data['code'])
                session.add(printing)
                seen_printing_ids.add(printing.id)


def create_set(set_data):
    """Given set data, create a CardSet."""
    card_set = models.CardSet(code=set_data['code'])
    card_set.name = set_data['name']
    card_set.block = set_data.get('block')
    card_set.release_date = datetime.datetime.strptime(
        set_data['releaseDate'], '%Y-%m-%d').date()
    card_set.type = set_data['type']
{'fraction_non_alphanumeric': 0.10563617954775566, 'fraction_numerical': 0.0, 'mean_word_length': 2.9362549800796813, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7613628', 'n_tokens_mistral': 922, 'n_tokens_neox': 899, 'n_words': 216}