<commit_before>import datetime
import requests
import socket
import sys
import time

from decimal import *
from urlparse import urljoin

try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET

try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

from django.conf import settings
from django.core.cache import cache
from django.core.urlresolvers import reverse
from django.db import connection, transaction, IntegrityError
from django.db.models import Count, Max

from thing import queries
from thing.models import *

from celery import task
from celery.exceptions import SoftTimeLimitExceeded
from celery.utils.log import get_task_logger
logger = get_task_logger(__name__)

from djcelery.models import TaskMeta

# ---------------------------------------------------------------------------
# Requests session
_session = requests.session(
    config={
        'pool_maxsize': 1,
        'max_retries': 0,
    },
    headers={
        'User-Agent': 'EVEthing-tasks (keep-alive)',
    },
)

# ---------------------------------------------------------------------------
# number of rows to request per WalletTransactions call, max is 2560
TRANSACTION_ROWS = 2560

API_KEY_INFO_URL = ('api_key_info', '/account/APIKeyInfo.xml.aspx', 'et_low')

CHAR_URLS = {
    APIKey.CHAR_ACCOUNT_STATUS_MASK: ('account_status', '/account/AccountStatus.xml.aspx', 'et_medium'),
    APIKey.CHAR_ASSET_LIST_MASK: ('asset_list', '/char/AssetList.xml.aspx', 'et_medium'),
    APIKey.CHAR_CHARACTER_SHEET_MASK: ('character_sheet', '/char/CharacterSheet.xml.aspx', 'et_medium'),
    APIKey.CHAR_CONTRACTS_MASK: ('contracts', '/char/Contracts.xml.aspx', 'et_medium'),
    APIKey.CHAR_CHARACTER_INFO_MASK: ('character_info', '/eve/CharacterInfo.xml.aspx', 'et_medium'),
    APIKey.CHAR_INDUSTRY_JOBS_MASK: ('industry_jobs', '/char/IndustryJobs.xml.aspx', 'et_medium'),
    APIKey.CHAR_MARKET_ORDERS_MASK: ('market_orders', '/char/MarketOrders.xml.aspx', 'et_medium'),
    APIKey.CHAR_SKILL_QUEUE_MASK: ('skill_queue', '/char/SkillQueue.xml.aspx', 'et_medium'),
    APIKey.CHAR_STANDINGS_MASK: ('standings', '/char/Standings.xml.aspx', 'et_medium'),
    APIKey.CHAR_WALLET_JOURNAL_MASK: ('wallet_journal', '/char/WalletJournal.xml.aspx', 'et_medium'),
    APIKey.CHAR_WALLET_TRANSACTIONS_MASK: ('wallet_transactions', '/char/WalletTransactions.xml.aspx', 'et_medium'),
}
CORP_URLS = {
    APIKey.CORP_ACCOUNT_BALANCE_MASK: ('account_balance', '/corp/AccountBalance.xml.aspx', 'et_medium'),
    APIKey.CORP_ASSET_LIST_MASK: ('asset_list', '/corp/AssetList.xml.aspx', 'et_medium'),
    APIKey.CORP_CONTRACTS_MASK: ('contracts', '/corp/Contracts.xml.aspx', 'et_medium'),
    APIKey.CORP_CORPORATION_SHEET_MASK: ('corporation_sheet', '/corp/CorporationSheet.xml.aspx', 'et_medium'),
    APIKey.CORP_INDUSTRY_JOBS_MASK: ('industry_jobs', '/corp/IndustryJobs.xml.aspx', 'et_medium'),
    APIKey.CORP_MARKET_ORDERS_MASK: ('market_orders', '/corp/MarketOrders.xml.aspx', 'et_medium'),
    APIKey.CORP_WALLET_JOURNAL_MASK: ('wallet_journal', '/corp/WalletJournal.xml.aspx', 'et_medium'),
    APIKey.CORP_WALLET_TRANSACTIONS_MASK: ('wallet_transactions', '/corp/WalletTransactions.xml.aspx', 'et_medium'),
}

# ---------------------------------------------------------------------------
# Class to wrap things
PENALTY_TIME = 12 * 60 * 60
PENALTY_MULT = 0.2

class APIJob:
    def __init__(self, apikey_id, taskstate_id):
        connection.queries = []
        self.start = time.time()
        self.api_total_time = 0.0

        self.ready = True

        # Fetch APIKey
        try:
            self.apikey = APIKey.objects.select_related('corp_character__corporation', 'user').get(pk=apikey_id)
        except APIKey.DoesNotExist:
            self.ready = False
        else:
            # Still valid?
            if not self.apikey.valid:
                self.ready = False

        # Fetch TaskState
        try:
            self.taskstate = TaskState.objects.get(pk=taskstate_id)
        except TaskState.DoesNotExist:
            self.ready = False

        self.root = None
        self.apicache = None
        self.padding = 0

    def completed(self):
        self.apicache.completed()
        self._taskstate_ready()

        if settings.DEBUG:# and False:
            print '%.3fs  %d queries (%.3fs)  API: %.2fs' % (time.time() - self.start,
                len(connection.queries), sum(float(q['time']) for q in connection.queries),
                self.api_total_time
            )
            for query in connection.queries:
                if query['sql'].startswith('INSERT INTO "thing_apicache"'):
                    print '%02.3fs  INSERT INTO "thing_apicache" ...' % (float(query['time']),)
                elif query['sql'].startswith('UPDATE "thing_apicache"'):
                    print '%02.3fs  UPDATE "thing_apicache" ...' % (float(query['time']),)
                else:
                    print '%02.3fs  %s' % (float(query['time']), query['sql'])

    def failed(self):
        self._taskstate_ready()

    def _taskstate_ready(self):
        now = datetime.datetime.now()
        self.taskstate.state = TaskState.READY_STATE
        self.taskstate.mod_time = now

        # If we have an APICache object, delay until the page is no longer cached
        if self.apicache is not None:
            self.taskstate.next_time = self.apicache.cached_until + datetime.timedelta(seconds=30)
        # No APICache? Just delay for 30 minutes
        else:
            self.taskstate.next_time = now + datetime.timedelta(minutes=30)

        #print 'before %s' % (self.taskstate.next_time)
        # if self.root:
        #     utcnow = datetime.datetime.utcnow()
        #     until = parse_api_date(self.root.find('cachedUntil').text)
        #     diff = until - utcnow
        #     self.taskstate.next_time = now + diff + datetime.timedelta(seconds=30 + self.padding)
        #     print self.padding
        #     self.taskstate.next_time = self.apicache.cached_until + datetime.timedelta(seconds=30)
        # else:
        #     # If we have an APICache object, delay until the page is no longer cached
        #     if self.apicache is not None:
        #         self.taskstate.next_time = self.apicache.cached_until + datetime.timedelta(seconds=30)
        #     # No APICache? Just delay for 30 minutes
        #     else:
        #         self.taskstate.next_time = now + datetime.timedelta(minutes=30)
        #print 'after %s' % (self.taskstate.next_time)

        self.taskstate.save()

    # ---------------------------------------------------------------------------
    # Perform an API request and parse the returned XML via ElementTree
    def fetch_api(self, url, params, use_auth=True, log_error=True):
        # Set the task to active
        self.taskstate.state = TaskState.ACTIVE_STATE
        self.taskstate.mod_time = datetime.datetime.now()
        self.taskstate.save()

        # Add the API key information
        if use_auth:
            params['keyID'] = self.apikey.keyid
            params['vCode'] = self.apikey.vcode

        # Check the API cache for this URL/params combo
        utcnow = datetime.datetime.utcnow()
        params_repr = repr(sorted(params.items()))
        
        # Retrieve the latest APICache object
        apicaches = list(APICache.objects.filter(url=url, parameters=params_repr, cached_until__gt=utcnow).order_by('-cached_until')[:1])
        
        # Data is not cached, fetch new data
        if len(apicaches) == 0:
            apicache = None
            
            # Fetch the URL
            full_url = urljoin(settings.API_HOST, url)
            start = time.time()
            try:
                #r = _session.post(full_url, params, prefetch=True)
                raise Exception
                data = r.text
            except Exception, e:
                _post_sleep(e)
                return False

            self.api_total_time += (time.time() - start)

            # If the status code is bad return False
            if not r.status_code == requests.codes.ok:
                _post_sleep('Bad status code: %s' % (r.status_code))
                return False

        # Data is cached, use that
        else:
            apicache = apicaches[0]
            data = apicache.text

        # Parse the data if there is any
        if data:<commit_msg>Probably helps if I take out the 'raise Exception' testing code :|<commit_after>import datetime
import requests
import socket
import sys
import time

from decimal import *
from urlparse import urljoin

try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET

try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

from django.conf import settings
from django.core.cache import cache
from django.core.urlresolvers import reverse
from django.db import connection, transaction, IntegrityError
from django.db.models import Count, Max

from thing import queries
from thing.models import *

from celery import task
from celery.exceptions import SoftTimeLimitExceeded
from celery.utils.log import get_task_logger
logger = get_task_logger(__name__)

from djcelery.models import TaskMeta

# ---------------------------------------------------------------------------
# Requests session
_session = requests.session(
    config={
        'pool_maxsize': 1,
        'max_retries': 0,
    },
    headers={
        'User-Agent': 'EVEthing-tasks (keep-alive)',
    },
)

# ---------------------------------------------------------------------------
# number of rows to request per WalletTransactions call, max is 2560
TRANSACTION_ROWS = 2560

API_KEY_INFO_URL = ('api_key_info', '/account/APIKeyInfo.xml.aspx', 'et_low')

CHAR_URLS = {
    APIKey.CHAR_ACCOUNT_STATUS_MASK: ('account_status', '/account/AccountStatus.xml.aspx', 'et_medium'),
    APIKey.CHAR_ASSET_LIST_MASK: ('asset_list', '/char/AssetList.xml.aspx', 'et_medium'),
    APIKey.CHAR_CHARACTER_SHEET_MASK: ('character_sheet', '/char/CharacterSheet.xml.aspx', 'et_medium'),
    APIKey.CHAR_CONTRACTS_MASK: ('contracts', '/char/Contracts.xml.aspx', 'et_medium'),
    APIKey.CHAR_CHARACTER_INFO_MASK: ('character_info', '/eve/CharacterInfo.xml.aspx', 'et_medium'),
    APIKey.CHAR_INDUSTRY_JOBS_MASK: ('industry_jobs', '/char/IndustryJobs.xml.aspx', 'et_medium'),
    APIKey.CHAR_MARKET_ORDERS_MASK: ('market_orders', '/char/MarketOrders.xml.aspx', 'et_medium'),
    APIKey.CHAR_SKILL_QUEUE_MASK: ('skill_queue', '/char/SkillQueue.xml.aspx', 'et_medium'),
    APIKey.CHAR_STANDINGS_MASK: ('standings', '/char/Standings.xml.aspx', 'et_medium'),
    APIKey.CHAR_WALLET_JOURNAL_MASK: ('wallet_journal', '/char/WalletJournal.xml.aspx', 'et_medium'),
    APIKey.CHAR_WALLET_TRANSACTIONS_MASK: ('wallet_transactions', '/char/WalletTransactions.xml.aspx', 'et_medium'),
}
CORP_URLS = {
    APIKey.CORP_ACCOUNT_BALANCE_MASK: ('account_balance', '/corp/AccountBalance.xml.aspx', 'et_medium'),
    APIKey.CORP_ASSET_LIST_MASK: ('asset_list', '/corp/AssetList.xml.aspx', 'et_medium'),
    APIKey.CORP_CONTRACTS_MASK: ('contracts', '/corp/Contracts.xml.aspx', 'et_medium'),
    APIKey.CORP_CORPORATION_SHEET_MASK: ('corporation_sheet', '/corp/CorporationSheet.xml.aspx', 'et_medium'),
    APIKey.CORP_INDUSTRY_JOBS_MASK: ('industry_jobs', '/corp/IndustryJobs.xml.aspx', 'et_medium'),
    APIKey.CORP_MARKET_ORDERS_MASK: ('market_orders', '/corp/MarketOrders.xml.aspx', 'et_medium'),
    APIKey.CORP_WALLET_JOURNAL_MASK: ('wallet_journal', '/corp/WalletJournal.xml.aspx', 'et_medium'),
    APIKey.CORP_WALLET_TRANSACTIONS_MASK: ('wallet_transactions', '/corp/WalletTransactions.xml.aspx', 'et_medium'),
}

# ---------------------------------------------------------------------------
# Class to wrap things
PENALTY_TIME = 12 * 60 * 60
PENALTY_MULT = 0.2

class APIJob:
    def __init__(self, apikey_id, taskstate_id):
        connection.queries = []
        self.start = time.time()
        self.api_total_time = 0.0

        self.ready = True

        # Fetch APIKey
        try:
            self.apikey = APIKey.objects.select_related('corp_character__corporation', 'user').get(pk=apikey_id)
        except APIKey.DoesNotExist:
            self.ready = False
        else:
            # Still valid?
            if not self.apikey.valid:
                self.ready = False

        # Fetch TaskState
        try:
            self.taskstate = TaskState.objects.get(pk=taskstate_id)
        except TaskState.DoesNotExist:
            self.ready = False

        self.root = None
        self.apicache = None
        self.padding = 0

    def completed(self):
        self.apicache.completed()
        self._taskstate_ready()

        if settings.DEBUG:# and False:
            print '%.3fs  %d queries (%.3fs)  API: %.2fs' % (time.time() - self.start,
                len(connection.queries), sum(float(q['time']) for q in connection.queries),
                self.api_total_time
            )
            for query in connection.queries:
                if query['sql'].startswith('INSERT INTO "thing_apicache"'):
                    print '%02.3fs  INSERT INTO "thing_apicache" ...' % (float(query['time']),)
                elif query['sql'].startswith('UPDATE "thing_apicache"'):
                    print '%02.3fs  UPDATE "thing_apicache" ...' % (float(query['time']),)
                else:
                    print '%02.3fs  %s' % (float(query['time']), query['sql'])

    def failed(self):
        self._taskstate_ready()

    def _taskstate_ready(self):
        now = datetime.datetime.now()
        self.taskstate.state = TaskState.READY_STATE
        self.taskstate.mod_time = now

        # If we have an APICache object, delay until the page is no longer cached
        if self.apicache is not None:
            self.taskstate.next_time = self.apicache.cached_until + datetime.timedelta(seconds=30)
        # No APICache? Just delay for 30 minutes
        else:
            self.taskstate.next_time = now + datetime.timedelta(minutes=30)

        #print 'before %s' % (self.taskstate.next_time)
        # if self.root:
        #     utcnow = datetime.datetime.utcnow()
        #     until = parse_api_date(self.root.find('cachedUntil').text)
        #     diff = until - utcnow
        #     self.taskstate.next_time = now + diff + datetime.timedelta(seconds=30 + self.padding)
        #     print self.padding
        #     self.taskstate.next_time = self.apicache.cached_until + datetime.timedelta(seconds=30)
        # else:
        #     # If we have an APICache object, delay until the page is no longer cached
        #     if self.apicache is not None:
        #         self.taskstate.next_time = self.apicache.cached_until + datetime.timedelta(seconds=30)
        #     # No APICache? Just delay for 30 minutes
        #     else:
        #         self.taskstate.next_time = now + datetime.timedelta(minutes=30)
        #print 'after %s' % (self.taskstate.next_time)

        self.taskstate.save()

    # ---------------------------------------------------------------------------
    # Perform an API request and parse the returned XML via ElementTree
    def fetch_api(self, url, params, use_auth=True, log_error=True):
        # Set the task to active
        self.taskstate.state = TaskState.ACTIVE_STATE
        self.taskstate.mod_time = datetime.datetime.now()
        self.taskstate.save()

        # Add the API key information
        if use_auth:
            params['keyID'] = self.apikey.keyid
            params['vCode'] = self.apikey.vcode

        # Check the API cache for this URL/params combo
        utcnow = datetime.datetime.utcnow()
        params_repr = repr(sorted(params.items()))
        
        # Retrieve the latest APICache object
        apicaches = list(APICache.objects.filter(url=url, parameters=params_repr, cached_until__gt=utcnow).order_by('-cached_until')[:1])
        
        # Data is not cached, fetch new data
        if len(apicaches) == 0:
            apicache = None
            
            # Fetch the URL
            full_url = urljoin(settings.API_HOST, url)
            start = time.time()
            try:
                r = _session.post(full_url, params, prefetch=True)
                data = r.text
            except Exception, e:
                _post_sleep(e)
                return False

            self.api_total_time += (time.time() - start)

            # If the status code is bad return False
            if not r.status_code == requests.codes.ok:
                _post_sleep('Bad status code: %s' % (r.status_code))
                return False

        # Data is cached, use that
        else:
            apicache = apicaches[0]
            data = apicache.text

        # Parse the data if there is any
        if data:
{'fraction_non_alphanumeric': 0.14483044818591415, 'fraction_numerical': 0.006165520512212474, 'mean_word_length': 2.3390736342042757, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 44}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7007880', 'n_tokens_mistral': 5334, 'n_tokens_neox': 5082, 'n_words': 1096}