<commit_before>import re
import math
from datetime import datetime, timedelta
import json
from django.views.generic import TemplateView
from models import Clan
import urllib2
from collections import OrderedDict, defaultdict
from django.conf import settings
from retrying import retry

import wargaming


wot = wargaming.WoT(settings.WARGAMING_KEY, language='ru', base_url='https://api.worldoftanks.ru/wot/')
wgn = wargaming.WGN(settings.WARGAMING_KEY, language='ru', base_url='https://api.worldoftanks.ru/wgn/')


class Province(object):
    def __init__(self, clan_id, battle_dict, province_dict):
        self._battle = battle_dict
        self._province = province_dict
        self._clan_id = clan_id

        self.province_id = self._province['province_id']
        self.arena_name = self._province['arena_name']
        self.province_name = self._province['province_name']
        self.prime_time = self._province['prime_time']
        self.prime_datetime = datetime.strptime(self._province['prime_time'], "%H:%M") \
            .replace(year=2016, second=0, microsecond=0)
        self.server = self._province['server']

        # Parse time
        self._province['battles_start_at'] = \
            datetime.strptime(self._province['battles_start_at'], '%Y-%m-%dT%H:%M:%S')

    @property
    def rounds_count(self):
        if not self._province['landing_type']:
            return 0

        if self._province['landing_type'] == 'tournament':
            count = len(self._province['competitors'])
        else:
            count = len(self._province['attackers'])
        if count > 0:
            return int(math.ceil(math.log(count, 2)))
        return -1

    @property
    def round_times(self):
        times = {}
        for battle_time, value in self.battle_times.items():
            minute = 30 if battle_time.minute >= 30 else 0
            times[battle_time.replace(minute=minute, second=0, microsecond=0)] = value
        return times

    @property
    def attack_type(self):
        if self._province['owner_clan_id'] == self._clan_id and (
            self._province['attackers'] or self._province['competitors']
        ):
            result = 'Defence'
        elif self._clan_id in self._province['attackers']:
            result = 'By land'
        elif self._clan_id in self._province['competitors']:
            result = 'Tournament'
        else:
            result = 'Unknown'
        return result

    @property
    def battle_times(self):
        rounds_count = self.rounds_count
        if rounds_count == -1:
            return {}

        battles_start_at = self._province['battles_start_at']
        if self._province['owner_clan_id'] == self._clan_id:
            times = {battles_start_at + timedelta(minutes=30) * rounds_count: 0}
        else:
            times = {}
            for i in range(rounds_count + 1):
                times[battles_start_at + timedelta(minutes=30) * i] = int(math.pow(2, rounds_count - i - 1))
        return times

    @property
    def url(self):
        return "https://ru.wargaming.net/globalmap" + self._province['uri']

    def has_battle_at(self, time):
        for bt in self.battle_times.keys():
            # 18:00:00 <= bt <= 18:29:59
            if time <= bt < time + timedelta(minutes=30):
                return True
        return False

    def at(self, time):
        res = self._province.copy()
        res['url'] = "https://ru.wargaming.net/globalmap" + res['uri']
        res['time'] = time
        return res

    def __repr__(self):
        return "<Battle for '%s' on '%s'>" % (self._province['province_id'], self._province['server'])


class ListBattles(TemplateView):
    template_name = 'list_battles.html'

    @staticmethod
    @retry(stop_max_attempt_number=5)
    def _clanprovinces(**kwargs):
        """wrapper to retry if WG servers has failed"""
        return wot.globalmap.clanprovinces(**kwargs)

    @staticmethod
    @retry(stop_max_attempt_number=5)
    def _provinces(**kwargs):
        """wrapper to retry if WG servers has failed"""
        return wot.globalmap.provinces(**kwargs)

    def get_context_data(self, **kwargs):
        context = super(ListBattles, self).get_context_data(**kwargs)
        clan_tag = self.request.GET.get('tag', 'SMIRK')
        clan = Clan.get_or_create(clan_tag=clan_tag)
        clan_id = clan.id

        response = urllib2.urlopen('https://ru.wargaming.net/globalmap/game_api/clan/%s/battles' % clan_id)
        data = json.loads(response.read())

        province_list = defaultdict(lambda: [])  # list of all provinces we have any actions, by front
        owned_provinces = []                     # list of all owned provinces
        provinces_data = {}                      # provinces data

        # prepare list of provinces to query
        for battle_type in ['battles', 'planned_battles']:
            for battle in data[battle_type]:
                province_list[battle['front_id']].append(battle['province_id'])

        # fetch for clan owned provinces
        for own in self._clanprovinces(clan_id=clan_id, language='ru')[str(clan_id)]:
            front_id = own['front_id']
            province_list[front_id].append(own['province_id'])
            owned_provinces.append(own['province_id'])

        # fetch for provinces data
        for front_id in province_list.keys():
            plist = list(set(province_list[front_id]))  # make list unique
            for p in self._provinces(front_id=front_id, province_id=plist):
                provinces_data[p['province_id']] = p

        # **** Battle section *****
        provinces = {}

        # fill battles from battles screen on GlobalMap
        for battle_type in ['battles', 'planned_battles']:
            for battle in data[battle_type]:
                provinces[battle['province_id']] = Province(clan_id, battle, provinces_data[battle['province_id']])

        # fill defences from owned battles
        for own in owned_provinces:
            if own not in provinces and (provinces_data[own]['attackers'] or provinces_data[own]['competitors']):
                provinces[own] = Province(clan_id, None, provinces_data[own])

        # fill attacks by land
        neighbours_list = defaultdict(lambda: [])
        for own in owned_provinces:
            for neighbor in provinces_data[own]['neighbours']:
                neighbours_list[provinces_data[own]['front_id']].append(neighbor)

        neighbours_data = {}
        for front_id in neighbours_list.keys():
            plist = list(set(neighbours_list[front_id]))  # make list unique
            for p in self._provinces(front_id=front_id, province_id=plist):
                neighbours_data[p['province_id']] = p

        for neighbor, data in neighbours_data.items():
            if clan_id in data['attackers']:  # attack performed by land
                provinces[neighbor] = Province(clan_id, None, data)

        start_time = min(set([i.prime_time for i in provinces.values()]))
        hour, minute = re.match('(\d{2}):(\d{2})', start_time).groups()

        day_start = (datetime.now()-timedelta(hours=10)) \
            .replace(hour=int(hour), minute=int(minute), second=0, microsecond=0)
        battle_matrix = OrderedDict()
        for i in range(16):
            battle_matrix[day_start + timedelta(minutes=30) * i] = []

        for time, battles_list in battle_matrix.items():
            for battle in provinces.values():
                if battle.has_battle_at(time):
                    battles_list.append(battle.at(time))

        context['battle_matrix'] = battle_matrix
        context['battle_matrix2'] = BattleMatrix(provinces)
        return context


class BattleMatrix(object):
    def __init__(self, provinces):
        self._provinces = provinces

    @property
    def times(self):
        res = []
        # print self._provinces
        for i in self._provinces.values():
            res.extend(i.round_times.keys())
        return sorted(set(res))

    @property
    def table(self):
        table = defaultdict(lambda: [])
        for province_id, prov in self._provinces.items():
            for time in self.times:
                item_in_row = {
                    'class': '',
                    'url': '',
                    'text': '',
                }
                if prov.has_battle_at(time):
                    round_id = prov.round_times[time]
                    if round_id == 1:
                        text = 'Final'
                    elif round_id == 0:
                        text = 'Owner'
                    else:
                        text = "1/%s" % round_id
                    data = {
                        'class': 'btn-default',
                        'text': text,
                        'url': prov.url
                    }
                    try:
                        table[prov].append(data)
                    except KeyError:
                        table[prov].append('Unknown')
                else:
                    table[prov].append(item_in_row)

        return sorted(table.items(), key=lambda k: (k[0].server, k[0].province_id))
<commit_msg>Add retry on api call, remove unused param<commit_after>import re
import math
from datetime import datetime, timedelta
import json
from django.views.generic import TemplateView
from models import Clan
import urllib2
from collections import OrderedDict, defaultdict
from django.conf import settings
from retrying import retry

import wargaming


wot = wargaming.WoT(settings.WARGAMING_KEY, language='ru', base_url='https://api.worldoftanks.ru/wot/')
wgn = wargaming.WGN(settings.WARGAMING_KEY, language='ru', base_url='https://api.worldoftanks.ru/wgn/')


class Province(object):
    def __init__(self, clan_id, province_dict):
        self._province = province_dict
        self._clan_id = clan_id

        self.province_id = self._province['province_id']
        self.arena_name = self._province['arena_name']
        self.province_name = self._province['province_name']
        self.prime_time = self._province['prime_time']
        self.prime_datetime = datetime.strptime(self._province['prime_time'], "%H:%M") \
            .replace(year=2016, second=0, microsecond=0)
        self.server = self._province['server']

        # Parse time
        self._province['battles_start_at'] = \
            datetime.strptime(self._province['battles_start_at'], '%Y-%m-%dT%H:%M:%S')

    @property
    def rounds_count(self):
        if not self._province['landing_type']:
            return 0

        if self._province['landing_type'] == 'tournament':
            count = len(self._province['competitors'])
        else:
            count = len(self._province['attackers'])
        if count > 0:
            return int(math.ceil(math.log(count, 2)))
        return -1

    @property
    def round_times(self):
        times = {}
        for battle_time, value in self.battle_times.items():
            minute = 30 if battle_time.minute >= 30 else 0
            times[battle_time.replace(minute=minute, second=0, microsecond=0)] = value
        return times

    @property
    def attack_type(self):
        if self._province['owner_clan_id'] == self._clan_id and (
            self._province['attackers'] or self._province['competitors']
        ):
            result = 'Defence'
        elif self._clan_id in self._province['attackers']:
            result = 'By land'
        elif self._clan_id in self._province['competitors']:
            result = 'Tournament'
        else:
            result = 'Unknown'
        return result

    @property
    def battle_times(self):
        rounds_count = self.rounds_count
        if rounds_count == -1:
            return {}

        battles_start_at = self._province['battles_start_at']
        if self._province['owner_clan_id'] == self._clan_id:
            times = {battles_start_at + timedelta(minutes=30) * rounds_count: 0}
        else:
            times = {}
            for i in range(rounds_count + 1):
                times[battles_start_at + timedelta(minutes=30) * i] = int(math.pow(2, rounds_count - i - 1))
        return times

    @property
    def url(self):
        return "https://ru.wargaming.net/globalmap" + self._province['uri']

    def has_battle_at(self, time):
        for bt in self.battle_times.keys():
            # 18:00:00 <= bt <= 18:29:59
            if time <= bt < time + timedelta(minutes=30):
                return True
        return False

    def at(self, time):
        res = self._province.copy()
        res['url'] = "https://ru.wargaming.net/globalmap" + res['uri']
        res['time'] = time
        return res

    def __repr__(self):
        return "<Battle for '%s' on '%s'>" % (self._province['province_id'], self._province['server'])


class ListBattles(TemplateView):
    template_name = 'list_battles.html'

    @staticmethod
    @retry(stop_max_attempt_number=5)
    def _clanprovinces(**kwargs):
        """wrapper to retry if WG servers has failed"""
        return wot.globalmap.clanprovinces(**kwargs)

    @staticmethod
    @retry(stop_max_attempt_number=5)
    def _provinces(**kwargs):
        """wrapper to retry if WG servers has failed"""
        return wot.globalmap.provinces(**kwargs)

    @staticmethod
    @retry(stop_max_attempt_number=5)
    def _data(clan_id):
        """wrapper to retry if WG servers has failed"""
        response = urllib2.urlopen('https://ru.wargaming.net/globalmap/game_api/clan/%s/battles' % clan_id)
        return json.loads(response.read())

    def get_context_data(self, **kwargs):
        context = super(ListBattles, self).get_context_data(**kwargs)
        clan_tag = self.request.GET.get('tag', 'SMIRK')
        clan = Clan.get_or_create(clan_tag=clan_tag)
        clan_id = clan.id
        province_list = defaultdict(lambda: [])  # list of all provinces we have any actions, by front
        owned_provinces = []                     # list of all owned provinces
        provinces_data = {}                      # provinces data

        data = self._data(clan_id)

        # prepare list of provinces to query
        for battle_type in ['battles', 'planned_battles']:
            for battle in data[battle_type]:
                province_list[battle['front_id']].append(battle['province_id'])

        # fetch for clan owned provinces
        clan_provinces = self._clanprovinces(clan_id=clan_id, language='ru')[str(clan_id)]
        if isinstance(clan_provinces, list):
            for own in clan_provinces:
                front_id = own['front_id']
                province_list[front_id].append(own['province_id'])
                owned_provinces.append(own['province_id'])

        # fetch for provinces data
        for front_id in province_list.keys():
            plist = list(set(province_list[front_id]))  # make list unique
            for p in self._provinces(front_id=front_id, province_id=plist):
                provinces_data[p['province_id']] = p

        # **** Battle section *****
        provinces = {}

        # fill battles from battles screen on GlobalMap
        for battle_type in ['battles', 'planned_battles']:
            for battle in data[battle_type]:
                province_id = battle['province_id']
                provinces[province_id] = Province(clan_id, provinces_data[province_id])

        # fill defences from owned battles
        for own in owned_provinces:
            if own not in provinces and (provinces_data[own]['attackers'] or provinces_data[own]['competitors']):
                provinces[own] = Province(clan_id, provinces_data[own])

        # fill attacks by land
        neighbours_list = defaultdict(lambda: [])
        for own in owned_provinces:
            for neighbor in provinces_data[own]['neighbours']:
                neighbours_list[provinces_data[own]['front_id']].append(neighbor)

        neighbours_data = {}
        for front_id in neighbours_list.keys():
            plist = list(set(neighbours_list[front_id]))  # make list unique
            for p in self._provinces(front_id=front_id, province_id=plist):
                neighbours_data[p['province_id']] = p

        for neighbor, data in neighbours_data.items():
            if clan_id in data['attackers']:  # attack performed by land
                provinces[neighbor] = Province(clan_id, data)

        start_time = min(set([i.prime_time for i in provinces.values()]))
        hour, minute = re.match('(\d{2}):(\d{2})', start_time).groups()

        day_start = (datetime.now()-timedelta(hours=10)) \
            .replace(hour=int(hour), minute=int(minute), second=0, microsecond=0)
        battle_matrix = OrderedDict()
        for i in range(16):
            battle_matrix[day_start + timedelta(minutes=30) * i] = []

        for time, battles_list in battle_matrix.items():
            for battle in provinces.values():
                if battle.has_battle_at(time):
                    battles_list.append(battle.at(time))

        context['battle_matrix'] = battle_matrix
        context['battle_matrix2'] = BattleMatrix(provinces)
        return context


class BattleMatrix(object):
    def __init__(self, provinces):
        self._provinces = provinces

    @property
    def times(self):
        res = []
        # print self._provinces
        for i in self._provinces.values():
            res.extend(i.round_times.keys())
        return sorted(set(res))

    @property
    def table(self):
        table = defaultdict(lambda: [])
        for province_id, prov in self._provinces.items():
            for time in self.times:
                item_in_row = {
                    'class': '',
                    'url': '',
                    'text': '',
                }
                if prov.has_battle_at(time):
                    round_id = prov.round_times[time]
                    if round_id == 1:
                        text = 'Final'
                    elif round_id == 0:
                        text = 'Owner'
                    else:
                        text = "1/%s" % round_id
                    data = {
                        'class': 'btn-default',
                        'text': text,
                        'url': prov.url
                    }
                    try:
                        table[prov].append(data)
                    except KeyError:
                        table[prov].append('Unknown')
                else:
                    table[prov].append(item_in_row)

        return sorted(table.items(), key=lambda k: (k[0].server, k[0].province_id))

{'fraction_non_alphanumeric': 0.11480981926085784, 'fraction_numerical': 0.006528189910979229, 'mean_word_length': 2.1892635925671025, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 10, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1414201', 'n_tokens_mistral': 5720, 'n_tokens_neox': 5463, 'n_words': 1162}