<commit_before>#!/usr/bin/python3
import sys
import json
import csv
import regex  # not re, because we need .captures()

def get_data(filename):
	with open(filename, newline='') as fp:
		yield from csv.DictReader(fp)

re_cost = regex.compile(r"^(?:o([WUBRGTXC]|\d+|cT))*$")
code_map = {'cT': "T"}
def cleancost(cost):
	parts = re_cost.match(cost)
	if not parts:
		raise ValueError("Could not parse cost: %r" % cost)
	parts = parts.captures(1)
	parts = (code_map.get(i, i) for i in parts)
	return "".join("{%s}" % i for i in parts)

re_italics = regex.compile(r"</?i>", regex.IGNORECASE)
re_textcost = regex.compile(r"\{([^{}]*)\}")
def cleantext(text):
	text = re_italics.sub('', text)
	text = re_textcost.sub(lambda match:cleancost(match.group(1)), text)
	return text

re_embalm = regex.compile(r"(?:^|\n|,)\s*Embalm\b", regex.IGNORECASE)
def getcard(row, setid):
	typeline = row['Card Type']
	if row['SuperType']:
		typeline = "%s %s" % (row['SuperType'], typeline)
	if row['SubType']:
		typeline = "%s \u2014 %s" % (typeline, row['SubType'])
	card = {
		'layout': 'normal',
		'name': row['Card Title'].replace('\u2019', "'"),
		'manaCost': cleancost(row['Mana']),
		'text': cleantext(row['Rules Text']),
		'type': typeline,
		'number': row['Collector Number'],
		'power': row['Power'],
		'toughness': row['Toughness'],
		'loyalty': row['Loyalty'],
	}
	card = dict((k, v.strip()) for k, v in card.items() if v is not None and v != "")
	yield card, setid

	# Create tokens for Embalm creatures for AKH preprere
	if re_embalm.search(card.get('text', '')):
		card = dict(card)
		card['internalname'] = card['name'] + "_TKN"
		card['name'] = card['name'] + " token"
		typeline = row['Card Type']
		if row['SuperType']:
			typeline = "%s %s" % (row['SuperType'], typeline)
		typeline = "%s \u2014 Zombie %s" % (typeline, row['SubType'])
		card['type'] = typeline
		del card['manaCost']
		del card['number']
		yield card, setid + "_TKN"

def getsplitcard(row, setid):
	# Format:
	#  Card Title is set to "Lefthalf /// Righthalf"
	#  Rules Text is set to "Left half rules///\nRighthalf\nRightcost\nRighttype\nRight half rules"
	names = row['Card Title'].split('///')
	if len(names) != 2:
		raise ValueError("Card has more than 2 names: %r" % row['Card Title'])
	names = [i.strip() for i in names]
	text = row['Rules Text'].split('///')
	if len(names) != 2:
		raise ValueError("Card has more than 2 texts: %r" % row['Card Title'])<commit_msg>Add eternalize tokens<commit_after>#!/usr/bin/python3
import sys
import json
import csv
import regex  # not re, because we need .captures()

def get_data(filename):
	with open(filename, newline='') as fp:
		yield from csv.DictReader(fp)

re_cost = regex.compile(r"^(?:o([WUBRGTXC]|\d+|cT))*$")
code_map = {'cT': "T"}
def cleancost(cost):
	parts = re_cost.match(cost)
	if not parts:
		raise ValueError("Could not parse cost: %r" % cost)
	parts = parts.captures(1)
	parts = (code_map.get(i, i) for i in parts)
	return "".join("{%s}" % i for i in parts)

re_italics = regex.compile(r"</?i>", regex.IGNORECASE)
re_textcost = regex.compile(r"\{([^{}]*)\}")
def cleantext(text):
	text = re_italics.sub('', text)
	text = re_textcost.sub(lambda match:cleancost(match.group(1)), text)
	return text

re_embalm = regex.compile(r"(?:^|\n|,)\s*(Embalm|Eternalize)\b", regex.IGNORECASE)
def getcard(row, setid):
	typeline = row['Card Type']
	if row['SuperType']:
		typeline = "%s %s" % (row['SuperType'], typeline)
	if row['SubType']:
		typeline = "%s \u2014 %s" % (typeline, row['SubType'])
	card = {
		'layout': 'normal',
		'name': row['Card Title'].replace('\u2019', "'"),
		'manaCost': cleancost(row['Mana']),
		'text': cleantext(row['Rules Text']),
		'type': typeline,
		'number': row['Collector Number'],
		'power': row['Power'],
		'toughness': row['Toughness'],
		'loyalty': row['Loyalty'],
	}
	card = dict((k, v.strip()) for k, v in card.items() if v is not None and v != "")
	yield card, setid

	# Create tokens for Embalm and Eternalize creatures for AKH/HOU preprere
	match = re_embalm.search(card.get('text', ''))
	if match:
		card = dict(card)
		card['internalname'] = card['name'] + "_TKN"
		card['name'] = card['name'] + " token"
		typeline = row['Card Type']
		if row['SuperType']:
			typeline = "%s %s" % (row['SuperType'], typeline)
		typeline = "%s \u2014 Zombie %s" % (typeline, row['SubType'])
		card['type'] = typeline
		del card['manaCost']
		del card['number']
		if match.group(1) == "Eternalize":
			card['power'] = card['toughness'] = '4'
		yield card, setid + "_TKN"

def getsplitcard(row, setid):
	# Format:
	#  Card Title is set to "Lefthalf /// Righthalf"
	#  Rules Text is set to "Left half rules///\nRighthalf\nRightcost\nRighttype\nRight half rules"
	names = row['Card Title'].split('///')
	if len(names) != 2:
		raise ValueError("Card has more than 2 names: %r" % row['Card Title'])
	names = [i.strip() for i in names]
	text = row['Rules Text'].split('///')
	if len(names) != 2:
		raise ValueError("Card has more than 2 texts: %r" % row['Card Title'])
{'fraction_non_alphanumeric': 0.20071899340922708, 'fraction_numerical': 0.00798881565807869, 'mean_word_length': 4.608062709966405, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7279659', 'n_tokens_mistral': 2007, 'n_tokens_neox': 1891, 'n_words': 499}