<commit_before>#!/usr/local/bin/python

import cPickle
import datetime
import os
import urllib2
import xml.etree.ElementTree as ET
from decimal import *
from urllib import urlencode

# Aurgh
from django.core.management import setup_environ
import settings
setup_environ(settings)

from thing.models import *


BASE_URL = 'http://api.eve-online.com'
CHARACTERS_URL = '%s/account/Characters.xml.aspx' % (BASE_URL)
ORDERS_URL = '%s/corp/MarketOrders.xml.aspx' % (BASE_URL)
TRANSACTIONS_CHAR_URL = '%s/char/WalletTransactions.xml.aspx' % (BASE_URL)
TRANSACTIONS_CORP_URL = '%s/corp/WalletTransactions.xml.aspx' % (BASE_URL)
WALLET_URL = '%s/corp/AccountBalance.xml.aspx' % (BASE_URL)

PADDING = datetime.timedelta(minutes=1)


def main():
	_now = datetime.datetime.now
	
	# Load cache
	cache_path = os.path.dirname(os.path.abspath(__file__))
	pickle_filepath = os.path.join(cache_path, 'cache/cache.pickle')
	if os.path.exists(pickle_filepath):
		cache = cPickle.load(open(pickle_filepath, 'rb'))
	else:
		cache = {
			'char': {},
			'corp': {},
		}
	
	
	# Make a character id map
	char_id_map = {}
	for character in Character.objects.all():
		char_id_map[character.eve_character_id] = character
	
	
	for character in Character.objects.all():
		# Skip if they have no valid user_id/api_key
		if not character.eve_user_id or len(character.eve_api_key) != 64:
			continue
		
		# Initialise cache
		if character.name not in cache['char']:
			cache['char'][character.name] = {
				'transactions': datetime.datetime(1900, 1, 1),
			}
		
		# Get their character ID if it has never been retrieved
		if not character.eve_character_id:
			root, times = fetch_api(CHARACTERS_URL, {}, character)
			err = root.find('error')
			if err is not None:
				show_error('character', err, times)
			else:
				for row in root.findall('result/rowset/row'):
					if row.attrib['name'].lower() == character.name.lower():
						character.eve_character_id = row.attrib['characterID']
						character.save()
		
		
		corporation = character.corporation
		
		
		# Character things
		# Update character transactions
		for wallet in CorpWallet.objects.filter(corporation=corporation):
			if _now() < cache['char'][character.name]['transactions']:
				continue
			
			params = {
				'characterID': character.eve_character_id,
			}
			
			wallet = CorpWallet.objects.get(corporation=character.corporation, account_key=1000)
			one_week_ago = None
			
			while True:
				root, times = fetch_api(TRANSACTIONS_CHAR_URL, params, character)
				err = root.find('error')
				if err is not None:
					# Fuck it, the API flat out lies about cache times
					if err.attrib['code'] not in ('101', '103'):
						show_error('chartrans', err, times)
					break
				
				# We need to stop asking for data if the oldest transaction entry is older
				# than one week
				if one_week_ago is None:
					one_week_ago = times['current'] - datetime.timedelta(7)
				
				rows = root.findall('result/rowset/row')
				if not rows:
					break
				
				for row in rows:
					transaction_time = parse_api_date(row.attrib['transactionDateTime'])
					<commit_msg>I have no idea what the fuck I was doing, fix this idiotic loop.<commit_after>#!/usr/local/bin/python

import cPickle
import datetime
import os
import urllib2
import xml.etree.ElementTree as ET
from decimal import *
from urllib import urlencode

# Aurgh
from django.core.management import setup_environ
import settings
setup_environ(settings)

from thing.models import *


BASE_URL = 'http://api.eve-online.com'
CHARACTERS_URL = '%s/account/Characters.xml.aspx' % (BASE_URL)
ORDERS_URL = '%s/corp/MarketOrders.xml.aspx' % (BASE_URL)
TRANSACTIONS_CHAR_URL = '%s/char/WalletTransactions.xml.aspx' % (BASE_URL)
TRANSACTIONS_CORP_URL = '%s/corp/WalletTransactions.xml.aspx' % (BASE_URL)
WALLET_URL = '%s/corp/AccountBalance.xml.aspx' % (BASE_URL)

PADDING = datetime.timedelta(minutes=1)


def main():
	_now = datetime.datetime.now
	
	# Load cache
	cache_path = os.path.dirname(os.path.abspath(__file__))
	pickle_filepath = os.path.join(cache_path, 'cache/cache.pickle')
	if os.path.exists(pickle_filepath):
		cache = cPickle.load(open(pickle_filepath, 'rb'))
	else:
		cache = {
			'char': {},
			'corp': {},
		}
	
	
	# Make a character id map
	char_id_map = {}
	for character in Character.objects.all():
		char_id_map[character.eve_character_id] = character
	
	
	for character in Character.objects.all():
		# Skip if they have no valid user_id/api_key
		if not character.eve_user_id or len(character.eve_api_key) != 64:
			continue
		
		# Initialise cache
		if character.name not in cache['char']:
			cache['char'][character.name] = {
				'transactions': datetime.datetime(1900, 1, 1),
			}
		
		# Get their character ID if it has never been retrieved
		if not character.eve_character_id:
			root, times = fetch_api(CHARACTERS_URL, {}, character)
			err = root.find('error')
			if err is not None:
				show_error('character', err, times)
			else:
				for row in root.findall('result/rowset/row'):
					if row.attrib['name'].lower() == character.name.lower():
						character.eve_character_id = row.attrib['characterID']
						character.save()
		
		
		corporation = character.corporation
		
		
		# Character things
		# Update character transactions
		if _now() >= cache['char'][character.name]['transactions'] and not character.eve_api_corp:
			params = {
				'characterID': character.eve_character_id,
			}
			
			wallet = CorpWallet.objects.get(corporation=character.corporation, account_key=1000)
			one_week_ago = None
			
			while True:
				root, times = fetch_api(TRANSACTIONS_CHAR_URL, params, character)
				err = root.find('error')
				if err is not None:
					# Fuck it, the API flat out lies about cache times
					if err.attrib['code'] not in ('101', '103'):
						show_error('chartrans', err, times)
					break
				
				# We need to stop asking for data if the oldest transaction entry is older
				# than one week
				if one_week_ago is None:
					one_week_ago = times['current'] - datetime.timedelta(7)
				
				rows = root.findall('result/rowset/row')
				if not rows:
					break
				
				for row in rows:
					transaction_time = parse_api_date(row.attrib['transactionDateTime'])
					
{'fraction_non_alphanumeric': 0.11683072454413426, 'fraction_numerical': 0.006777472970792319, 'mean_word_length': 3.771362586605081, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 1, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6222515', 'n_tokens_mistral': 2443, 'n_tokens_neox': 2030, 'n_words': 497}