<commit_before>#!/usr/bin/env python3
"""
This script downloads the latest MTG card data from http://mtgjson.com/ and processes
it to turn the highly-structured data there into a flat list of card names to descriptions
formatted to send down the chat.
"""

import sys
import os
import urllib.request
import urllib.error
import contextlib
import time
import zipfile
import io
import json
import re
import datetime
import dateutil.parser
import sqlalchemy

from common import utils
import common.postgres
from common.cardname import clean_text

URL = 'http://mtgjson.com/json/AllSets.json.zip'
ZIP_FILENAME = 'AllSets.json.zip'
SOURCE_FILENAME = 'AllSets.json'
MAXLEN = 450

engine, metadata = common.postgres.new_engine_and_metadata()

def main():
	if not do_download_file(URL, ZIP_FILENAME):
		print("No new version of mtgjson data file")
		return

	print("Reading card data...")
	with zipfile.ZipFile(ZIP_FILENAME) as zfp:
		fp = io.TextIOWrapper(zfp.open(SOURCE_FILENAME))
		mtgjson = json.load(fp)

	with open("extracards.json") as fp:
		# If the set is in both mtgjson and the extra data, use the one from mtgjson
		allcards = json.load(fp)
		allcards.update(mtgjson)
		mtgjson = allcards

	print("Processing...")
	cards = metadata.tables["cards"]
	card_multiverse = metadata.tables["card_multiverse"]
	card_collector = metadata.tables["card_collector"]
	with engine.begin() as conn:
		conn.execute(card_multiverse.delete())
		conn.execute(card_collector.delete())
		conn.execute(cards.delete())
		cardid = 0
		for setid, expansion in mtgjson.items():
			release_date = dateutil.parser.parse(expansion['releaseDate']).date()
			for card in expansion['cards']:
				cardid += 1
				if card['layout'] in ('token', 'plane', 'scheme', 'phenomenon', 'vanguard'):  # don't care about these special cards for now
					continue
				if card['name'] == 'B.F.M. (Big Furry Monster)':  # do this card special
					continue

				cardname, description, multiverseids, collector = process_card(card, expansion)
				if description is None:
					continue

				# Check if there's already a row for this card in the DB
				# (keep the one with the latest release date - it's more likely to have the accurate text in mtgjson)
				rows = conn.execute(sqlalchemy.select([cards.c.id, cards.c.lastprinted])
					.where(cards.c.filteredname == cardname)).fetchall()
				if not rows:
					real_cardid = cardid
					conn.execute(cards.insert(),
						id=real_cardid,
						filteredname=cardname,
						name=card['name'],
						text=description,
						lastprinted=release_date,
					)
				elif rows[0][1] < release_date:
					real_cardid = rows[0][0]
					conn.execute(cards.update().where(cards.c.id == real_cardid),
						name=card["name"],
						text=description,
						lastprinted=release_date,
					)
				else:
					real_cardid = rows[0][0]

				for mid in multiverseids:
					rows = conn.execute(sqlalchemy.select([card_multiverse.c.cardid])
						.where(card_multiverse.c.id == mid)).fetchall()
					if not rows:
						conn.execute(card_multiverse.insert(),
							id=mid,
							cardid=real_cardid,
						)
					elif rows[0][0] != real_cardid:
						rows2 = conn.execute(sqlalchemy.select([cards.c.name]).where(cards.c.id == rows[0][0])).fetchall()
						print("Different names for multiverseid %d: \"%s\" and \"%s\"" % (mid, card['name'], rows2[0][0]))

				if collector:
					rows = conn.execute(sqlalchemy.select([card_collector.c.cardid])
						.where((card_collector.c.setid == setid) & (card_collector.c.collector == collector))).fetchall()
					if not rows:
						conn.execute(card_collector.insert(),
							setid=setid,
							collector=collector,
							cardid=real_cardid,
						)
					elif rows[0][0] != real_cardid:
						rows2 = conn.execute(sqlalchemy.select([cards.c.name]).where(cards.c.id == rows[0][0])).fetchall()
						print("Different names for set %s collector number %s: \"%s\" and \"%s\"" % (setid, collector, card['name'], rows2[0][0]))

		cardid += 1
		conn.execute(cards.insert(),
			id=cardid,
			filteredname="bfmbigfurrymonster",
			name="B.F.M. (Big Furry Monster)""B.F.M. (Big Furry Monster)",
			text="B.F.M. (Big Furry Monster) (BBBBBBBBBBBBBBB) | Summon \u2014 The Biggest, Baddest, Nastiest, Scariest Creature You'll Ever See [99/99] | You must play both B.F.M. cards to put B.F.M. into play. If either B.F.M. card leaves play, sacrifice the other. / B.F.M. can only be blocked by three or more creatures.",
			lastprinted=datetime.date(1998, 8, 11),
		)
		conn.execute(card_multiverse.insert(), [
			{"id": 9780, "cardid": cardid},
			{"id": 9844, "cardid": cardid},
		])
		conn.execute(card_collector.insert(), [
			{"setid": "UGL", "collector": "28", "cardid": cardid},
			{"setid": "UGL", "collector": "29", "cardid": cardid},
		])

def do_download_file(url, fn):
	"""
	Download a file, checking that there is a new version of the file on the
	server before doing so. Returns True if a download occurs.
	"""
	# Much of this code cribbed from urllib.request.urlretrieve, with If-Modified-Since logic added

	req = urllib.request.Request(url)
	try:
		stat = os.stat(fn)
	except FileNotFoundError:
		pass
	else:
		mtime = time.strftime('%a, %d %b %Y %H:%M:%S %z', time.gmtime(stat.st_mtime))
		req.add_header('If-Modified-Since', mtime)

	try:
		fp = urllib.request.urlopen(req)
	except urllib.error.HTTPError as e:
		if e.code == 304: # Not Modified
			return False
		else:
			raise<commit_msg>Tweak build-card script<commit_after>#!/usr/bin/env python3
"""
This script downloads the latest MTG card data from http://mtgjson.com/ and processes
it to turn the highly-structured data there into a flat list of card names to descriptions
formatted to send down the chat.
"""

import sys
import os
import urllib.request
import urllib.error
import contextlib
import time
import zipfile
import io
import json
import re
import datetime
import dateutil.parser
import sqlalchemy

from common import utils
import common.postgres
from common.cardname import clean_text

URL = 'http://mtgjson.com/json/AllSets.json.zip'
ZIP_FILENAME = 'AllSets.json.zip'
SOURCE_FILENAME = 'AllSets.json'
MAXLEN = 450

engine, metadata = common.postgres.new_engine_and_metadata()

def main():
	if not do_download_file(URL, ZIP_FILENAME):
		print("No new version of mtgjson data file")
		return

	print("Reading card data...")
	with zipfile.ZipFile(ZIP_FILENAME) as zfp:
		fp = io.TextIOWrapper(zfp.open(SOURCE_FILENAME))
		mtgjson = json.load(fp)

	try:
		with open("extracards.json") as fp:
			extracards = json.load(fp)
	except IOError:
		pass
	else:
		# If the set is in both mtgjson and the extra data, use the one from mtgjson
		extracards.update(mtgjson)
		mtgjson = extracards
		del extracards

	print("Processing...")
	cards = metadata.tables["cards"]
	card_multiverse = metadata.tables["card_multiverse"]
	card_collector = metadata.tables["card_collector"]
	with engine.begin() as conn:
		conn.execute(card_multiverse.delete())
		conn.execute(card_collector.delete())
		conn.execute(cards.delete())
		cardid = 0
		for setid, expansion in mtgjson.items():
			release_date = dateutil.parser.parse(expansion['releaseDate']).date()
			for card in expansion['cards']:
				cardid += 1
				if card['layout'] in ('token', 'plane', 'scheme', 'phenomenon', 'vanguard'):  # don't care about these special cards for now
					continue
				if card['name'] == 'B.F.M. (Big Furry Monster)':  # do this card special
					continue

				cardname, description, multiverseids, collector = process_card(card, expansion)
				if description is None:
					continue

				# Check if there's already a row for this card in the DB
				# (keep the one with the latest release date - it's more likely to have the accurate text in mtgjson)
				rows = conn.execute(sqlalchemy.select([cards.c.id, cards.c.lastprinted])
					.where(cards.c.filteredname == cardname)).fetchall()
				if not rows:
					real_cardid = cardid
					conn.execute(cards.insert(),
						id=real_cardid,
						filteredname=cardname,
						name=card['name'],
						text=description,
						lastprinted=release_date,
					)
				elif rows[0][1] < release_date:
					real_cardid = rows[0][0]
					conn.execute(cards.update().where(cards.c.id == real_cardid),
						name=card["name"],
						text=description,
						lastprinted=release_date,
					)
				else:
					real_cardid = rows[0][0]

				for mid in multiverseids:
					rows = conn.execute(sqlalchemy.select([card_multiverse.c.cardid])
						.where(card_multiverse.c.id == mid)).fetchall()
					if not rows:
						conn.execute(card_multiverse.insert(),
							id=mid,
							cardid=real_cardid,
						)
					elif rows[0][0] != real_cardid:
						rows2 = conn.execute(sqlalchemy.select([cards.c.name]).where(cards.c.id == rows[0][0])).fetchall()
						print("Different names for multiverseid %d: \"%s\" and \"%s\"" % (mid, card['name'], rows2[0][0]))

				if collector:
					rows = conn.execute(sqlalchemy.select([card_collector.c.cardid])
						.where((card_collector.c.setid == setid) & (card_collector.c.collector == collector))).fetchall()
					if not rows:
						conn.execute(card_collector.insert(),
							setid=setid,
							collector=collector,
							cardid=real_cardid,
						)
					elif rows[0][0] != real_cardid:
						rows2 = conn.execute(sqlalchemy.select([cards.c.name]).where(cards.c.id == rows[0][0])).fetchall()
						print("Different names for set %s collector number %s: \"%s\" and \"%s\"" % (setid, collector, card['name'], rows2[0][0]))

		cardid += 1
		conn.execute(cards.insert(),
			id=cardid,
			filteredname="bfmbigfurrymonster",
			name="B.F.M. (Big Furry Monster)""B.F.M. (Big Furry Monster)",
			text="B.F.M. (Big Furry Monster) (BBBBBBBBBBBBBBB) | Summon \u2014 The Biggest, Baddest, Nastiest, Scariest Creature You'll Ever See [99/99] | You must play both B.F.M. cards to put B.F.M. into play. If either B.F.M. card leaves play, sacrifice the other. / B.F.M. can only be blocked by three or more creatures.",
			lastprinted=datetime.date(1998, 8, 11),
		)
		conn.execute(card_multiverse.insert(), [
			{"id": 9780, "cardid": cardid},
			{"id": 9844, "cardid": cardid},
		])
		conn.execute(card_collector.insert(), [
			{"setid": "UGL", "collector": "28", "cardid": cardid},
			{"setid": "UGL", "collector": "29", "cardid": cardid},
		])

def do_download_file(url, fn):
	"""
	Download a file, checking that there is a new version of the file on the
	server before doing so. Returns True if a download occurs.
	"""
	# Much of this code cribbed from urllib.request.urlretrieve, with If-Modified-Since logic added

	req = urllib.request.Request(url, headers={
		'User-Agent': "LRRbot/2.0 (https://lrrbot.mrphlip.com/)",
	})
	try:
		stat = os.stat(fn)
	except FileNotFoundError:
		pass
	else:
		mtime = time.strftime('%a, %d %b %Y %H:%M:%S %z', time.gmtime(stat.st_mtime))
		req.add_header('If-Modified-Since', mtime)

	try:
		fp = urllib.request.urlopen(req)
	except urllib.error.HTTPError as e:
		if e.code == 304: # Not Modified
			return False
		else:
			raise
{'fraction_non_alphanumeric': 0.13782109673893242, 'fraction_numerical': 0.01093095281472035, 'mean_word_length': 3.7528138528138526, 'pattern_counts': {'":': 20, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7411896', 'n_tokens_mistral': 4450, 'n_tokens_neox': 3727, 'n_words': 982}