<commit_before># coding=UTF-8
__author__ = 'fantomen'
import csv
from difflib import SequenceMatcher
import urllib
import datetime
import numpy as np


class fbDataParser:
    NAMES = {
        'Manchester United': 'Man United',
        'Manchester City': 'Man City',
        'Queens PR': 'QPR',
        'West Bromwich': 'West Brom',
        'W Bromwich': 'West Brom',
        'Oldham Athletic': 'Oldham',
        'Bayern München': 'Bayern Munich',
        'B München': 'Bayern Munich',
        'Bor<NAME>': 'M\'gladbach',
        'Mönchengl': 'M\'gladbach',
        '<NAME>': 'Crewe',
        'Fleetwood': 'Fleetwood Town',
        'Preston North End': 'Preston',
        'Queens Park Rangers': 'QPR',
        'Wolverhampton': 'Wolves',
        'Yeovil Town': 'Yeovil',
        'Nottingham Forest': 'Nott\'m Forest',
        'Athletic Bilbao': 'Ath Bilbao',
        'Sporting Gijon': 'Sp Gijon',
        'Real Sociedad': 'Sociedad',
        'Deportivo La Coruna': 'La Coruna',
        'Atletico de Madrid': 'Ath Madrid',
        '<NAME>': 'Vallecano',
        'Cel<NAME>': 'Celta',
        'Bayer Leverkusen': 'Leverkusen',
        '1899 Hoffenheim': 'Hoffenheim',
        'Hamburger SV': 'Hamburg',
        'Djurgårdens IF': 'Djurgården',
        'Assyriska Föreningen': 'Assyriska',
        'Örebro SK': 'Örebro',
        'Real Betis': 'Betis',
        'Racing Santander': 'Santander',
        'Borussia Dortmund': 'Dortmund',
        '<NAME>': 'Hertha',
        'H Berlin': 'Hertha',
        'FSV Mainz': 'Mainz',
        'FC Nürnberg': 'Nürnberg',
        'Eintracht Braunschweig': 'Braunschweig',
        'Olympique Marseille': 'Marseille',
        'FC Metz': 'Metz',
        'VfL Bochum': 'Bochum',
        'Glasgow Rangers': 'Rangers',
        '<NAME>': 'Partick',
        'Energie Cottbus': 'Cottbus',
        '<NAME>': 'Bielefeld',
        'FC St. Pauli': 'St Pauli',
        'Dundee FC': 'Dundee',
        'Jönköpings Södra IF': 'Jönköpings Södra',
        'Swindon Town': 'Swindon',
        'Scunthorpe United': 'Scunthorpe',
        'Coventry City': 'Coventry',
        'Colchester United': 'Colchester',
        'Carlisle United': 'Carlisle',
        'Stevenage Football Club': 'Stevenage',
        'Tranmere Rovers': 'Tranmere',
        'Hartlepool United': 'Hartlepool',
        'Shrewsbury Town': 'Shrewsbury',
        'FC Köln': 'FC Koln',
        'Almería': 'Almeria',
        'Tarragona': 'Gimnastic',
        'Dalkurd FF': 'Dalkurd',
        'SV Darmstadt 98': 'Darmstadt',
        'GFC Ajaccio': 'Ajaccio GFCO',
        'ES Troyes AC': 'Troyes',
        'Viking Stavanger': 'Viking FK',
        'Odd Grenland': 'Odd',
        'Crystal P': 'Crystal Palace',
        'Paris Saint Germain': 'Paris SG',
        'AZ Alkmaar': 'AZ',
        'Twente': 'FC Twente',
        'PSV Eindhoven': 'PSV',
        'Cambuur': 'SC <NAME>',
        'Nijmegen': 'N.E.C.',
        'Roda': 'Roda JC Kerkrade',
        'V<NAME>': 'Vitesse',
        'HJK Helsinki': 'HJK',
        '<NAME>': 'RoPS',
        'FC Inter Turku': 'FC Inter',
        'Seinajoen JK': 'SJK',
        'Vaasa PS': 'VPS',
        'Kuopio PS': 'KuPS',
        'FC Ilves': 'Ilves',
        'PS Kemi Kings': 'PS Kemi',
        'Halmstads BK': 'Halmstad',
        'Århus' : 'AGF Aarhus',
        'Ålborg': 'AaB Ålborg',
        'Odense': 'Odense Boldklub',
        'Viborg': 'Viborg FF',
        'Sönderjyske Fodbold': 'SønderjyskE'

    }

    LEAGUES = [
        'http://www.football-data.co.uk/mmz4281/1516/E0.csv',
        'http://www.football-data.co.uk/mmz4281/1415/E0.csv',
        'http://www.football-data.co.uk/mmz4281/1314/E0.csv',
        'http://www.football-data.co.uk/mmz4281/1516/E1.csv',
        'http://www.football-data.co.uk/mmz4281/1415/E1.csv',
        'http://www.football-data.co.uk/mmz4281/1314/E1.csv',
        'http://www.football-data.co.uk/mmz4281/1516/I1.csv',
        'http://www.football-data.co.uk/mmz4281/1415/I1.csv',
        'http://www.football-data.co.uk/mmz4281/1314/I1.csv',
        'http://www.football-data.co.uk/mmz4281/1516/SP1.csv',
        'http://www.football-data.co.uk/mmz4281/1415/SP1.csv',
        'http://www.football-data.co.uk/mmz4281/1314/SP1.csv',
        'http://www.football-data.co.uk/mmz4281/1516/D1.csv',<commit_msg>strömsgodset<commit_after># coding=UTF-8
__author__ = 'fantomen'
import csv
from difflib import SequenceMatcher
import urllib
import datetime
import numpy as np


class fbDataParser:
    NAMES = {
        'Manchester United': 'Man United',
        'Manchester City': 'Man City',
        'Queens PR': 'QPR',
        'West Bromwich': 'West Brom',
        'W Bromwich': 'West Brom',
        'Oldham Athletic': 'Oldham',
        'Bayern München': 'Bayern Munich',
        'B München': 'Bayern Munich',
        'Borussia Mönchengladbach': 'M\'gladbach',
        'Mönchengl': 'M\'gladbach',
        'Cre<NAME>': 'Crewe',
        'Fleetwood': 'Fleetwood Town',
        'Preston North End': 'Preston',
        'Queens Park Rangers': 'QPR',
        'Wolverhampton': 'Wolves',
        'Yeovil Town': 'Yeovil',
        'Nottingham Forest': 'Nott\'m Forest',
        'Athletic Bilbao': 'Ath Bilbao',
        'Sporting Gijon': 'Sp Gijon',
        'Real Sociedad': 'Sociedad',
        'Deportivo La Coruna': 'La Coruna',
        'Atletico de Madrid': 'Ath Madrid',
        'Rayo Vallecano': 'Vallecano',
        '<NAME>': 'Celta',
        '<NAME>en': 'Leverkusen',
        '1899 Hoffenheim': 'Hoffenheim',
        'Hamburger SV': 'Hamburg',
        'Djurgårdens IF': 'Djurgården',
        'Ass<NAME>': 'Assyriska',
        'Örebro SK': 'Örebro',
        'Real Betis': 'Betis',
        'Racing Santander': 'Santander',
        'Bor<NAME>': 'Dortmund',
        '<NAME>': 'Hertha',
        'H Berlin': 'Hertha',
        'FSV Mainz': 'Mainz',
        'FC Nürnberg': 'Nürnberg',
        'Eintracht Braunschweig': 'Braunschweig',
        'Olympique Marseille': 'Marseille',
        'FC Metz': 'Metz',
        'VfL Bochum': 'Bochum',
        'Glasgow Rangers': 'Rangers',
        '<NAME>': 'Partick',
        'Energie Cottbus': 'Cottbus',
        '<NAME>': 'Bielefeld',
        'FC St. Pauli': 'St Pauli',
        'Dundee FC': 'Dundee',
        'Jönköpings Södra IF': 'Jönköpings Södra',
        'Swindon Town': 'Swindon',
        'Scunthorpe United': 'Scunthorpe',
        'Coventry City': 'Coventry',
        'Colchester United': 'Colchester',
        'Carlisle United': 'Carlisle',
        'Stevenage Football Club': 'Stevenage',
        'Tranmere Rovers': 'Tranmere',
        'Hartlepool United': 'Hartlepool',
        'Shrewsbury Town': 'Shrewsbury',
        'FC Köln': 'FC Koln',
        'Almería': 'Almeria',
        'Tarragona': 'Gimnastic',
        'Dalkurd FF': 'Dalkurd',
        'SV Darmstadt 98': 'Darmstadt',
        'GFC Ajaccio': 'Ajaccio GFCO',
        'ES Troyes AC': 'Troyes',
        'Viking Stavanger': 'Viking FK',
        'Odd Grenland': 'Odd',
        'Crystal P': 'Crystal Palace',
        'Paris Saint Germain': 'Paris SG',
        'AZ Alkmaar': 'AZ',
        'Twente': 'FC Twente',
        'PSV Eindhoven': 'PSV',
        'Cambuur': 'SC Cambuur Leeuwarden',
        'Nijmegen': 'N.E.C.',
        'Roda': 'Roda JC Kerkrade',
        'Vitesse Arnheim': 'Vitesse',
        'HJK Helsinki': 'HJK',
        'Rovaniemi Palloseura': 'RoPS',
        'FC Inter Turku': 'FC Inter',
        'Seinajoen JK': 'SJK',
        'Vaasa PS': 'VPS',
        'Kuopio PS': 'KuPS',
        'FC Ilves': 'Ilves',
        'PS Kemi Kings': 'PS Kemi',
        'Halmstads BK': 'Halmstad',
        'Århus' : 'AGF Aarhus',
        'Ålborg': 'AaB Ålborg',
        'Odense': 'Odense Boldklub',
        'Viborg': 'Viborg FF',
        'Sönderjyske Fodbold': 'SønderjyskE',
        'Strömsgodset Toppfotball': 'Strömsgodset'

    }

    LEAGUES = [
        'http://www.football-data.co.uk/mmz4281/1516/E0.csv',
        'http://www.football-data.co.uk/mmz4281/1415/E0.csv',
        'http://www.football-data.co.uk/mmz4281/1314/E0.csv',
        'http://www.football-data.co.uk/mmz4281/1516/E1.csv',
        'http://www.football-data.co.uk/mmz4281/1415/E1.csv',
        'http://www.football-data.co.uk/mmz4281/1314/E1.csv',
        'http://www.football-data.co.uk/mmz4281/1516/I1.csv',
        'http://www.football-data.co.uk/mmz4281/1415/I1.csv',
        'http://www.football-data.co.uk/mmz4281/1314/I1.csv',
        'http://www.football-data.co.uk/mmz4281/1516/SP1.csv',
        'http://www.football-data.co.uk/mmz4281/1415/SP1.csv',
        'http://www.football-data.co.uk/mmz4281/1314/SP1.csv',
        'http://www.football-data.co.uk/mmz4281/1516/D1.csv',
{'fraction_non_alphanumeric': 0.17757554544393886, 'fraction_numerical': 0.02893478007233695, 'mean_word_length': 2.28680981595092, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 26, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5239039', 'n_tokens_mistral': 3526, 'n_tokens_neox': 3296, 'n_words': 646}