<commit_before># whatlastgenre
# Improves genre metadata of audio files
# based on tags from various music sites.
#
# Copyright (c) 2012-2015 YetAnotherNerd
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge,
# publish, distribute, sublicense, and/or sell copies of the Software,
# and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

'''whatlastgenre dataprovider'''

from __future__ import division, print_function

from collections import defaultdict
import json
import logging
import os.path
from tempfile import NamedTemporaryFile
import time

import requests

from wlg import __version__


try:  # use optional requests_cache if available
    import requests_cache
    requests_cache.install_cache(
        os.path.expanduser('~/.whatlastgenre/reqcache'),
        expire_after=180 * 24 * 60 * 60,
        allowable_methods=('GET', 'POST'),
        old_data_on_error=True)
except ImportError:
    requests_cache = None


HEADERS = {'User-Agent': "whatlastgenre/%s" % __version__}


def get_daprs(conf):
    '''Return a list of DataProviders activated in the conf file.'''
    sources = conf.get_list('wlg', 'sources')
    daprs = []
    if 'whatcd' in sources:
        cred = {'username': conf.get('wlg', 'whatcduser'),
                'password': conf.get('wlg', '<PASSWORD>')}
        if all(cred.itervalues()):
            daprs.append(WhatCD(cred))
        else:
            print("No What.CD credentials specified. "
                  "What.CD support disabled.\n")
    if 'lastfm' in sources:
        daprs.append(LastFM())
    if 'discogs' in sources:
        daprs.append(Discogs())
    if 'mbrainz' in sources:
        daprs.append(MusicBrainz())
    if 'echonest' in sources:
        daprs.append(EchoNest())
    if not daprs:
        print("Where do you want to get your data from?\nAt least one "
              "source must be activated (multiple sources recommended)!")
        exit()
    return daprs


class Cache(object):
    '''Load and save a dict as json from/into a file for some
    speedup.
    '''

    def __init__(self, path, update_cache):
        self.fullpath = os.path.join(path, 'cache')
        self.update_cache = update_cache
        self.expire_after = 180 * 24 * 60 * 60
        self.time = time.time()
        self.dirty = False
        self.cache = {}
        self.new = set()
        try:
            with open(self.fullpath) as file_:
                self.cache = json.load(file_)
        except (IOError, ValueError):
            pass

        # backward compatibility code
        oldkeys = {k: v for k, v in self.cache.iteritems() if '##' in k}
        for key, val in oldkeys.iteritems():
            if val['data']:
                # only keep some keys
                keep = ['info', 'year'] if len(val['data']) > 1 else []
                val['data'] = [{k: v for k, v in d.iteritems()
                                if k in ['tags', 'releasetype'] + keep}
                               for d in val['data'] if d]
                # all tag data are dicts now
                for dat in val['data']:
                    if dat and 'tags' in dat and isinstance(dat['tags'], list):
                        dat['tags'] = {t: 0 for t in dat['tags']}
            del self.cache[key]
            key = str(tuple(key.encode("utf-8").split('##')))
            val['data'] = val['data'] if val['data'] else []
            self.cache[key] = (val['time'], val['data'])
            self.dirty = True

    def __del__(self):
        self.save()
<commit_msg>remove backward compatibility cache code<commit_after># whatlastgenre
# Improves genre metadata of audio files
# based on tags from various music sites.
#
# Copyright (c) 2012-2015 YetAnotherNerd
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge,
# publish, distribute, sublicense, and/or sell copies of the Software,
# and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

'''whatlastgenre dataprovider'''

from __future__ import division, print_function

from collections import defaultdict
import json
import logging
import os.path
from tempfile import NamedTemporaryFile
import time

import requests

from wlg import __version__


try:  # use optional requests_cache if available
    import requests_cache
    requests_cache.install_cache(
        os.path.expanduser('~/.whatlastgenre/reqcache'),
        expire_after=180 * 24 * 60 * 60,
        allowable_methods=('GET', 'POST'),
        old_data_on_error=True)
except ImportError:
    requests_cache = None


HEADERS = {'User-Agent': "whatlastgenre/%s" % __version__}


def get_daprs(conf):
    '''Return a list of DataProviders activated in the conf file.'''
    sources = conf.get_list('wlg', 'sources')
    daprs = []
    if 'whatcd' in sources:
        cred = {'username': conf.get('wlg', 'whatcduser'),
                'password': conf.get('wlg', '<PASSWORD>')}
        if all(cred.itervalues()):
            daprs.append(WhatCD(cred))
        else:
            print("No What.CD credentials specified. "
                  "What.CD support disabled.\n")
    if 'lastfm' in sources:
        daprs.append(LastFM())
    if 'discogs' in sources:
        daprs.append(Discogs())
    if 'mbrainz' in sources:
        daprs.append(MusicBrainz())
    if 'echonest' in sources:
        daprs.append(EchoNest())
    if not daprs:
        print("Where do you want to get your data from?\nAt least one "
              "source must be activated (multiple sources recommended)!")
        exit()
    return daprs


class Cache(object):
    '''Load and save a dict as json from/into a file for some
    speedup.
    '''

    def __init__(self, path, update_cache):
        self.fullpath = os.path.join(path, 'cache')
        self.update_cache = update_cache
        self.expire_after = 180 * 24 * 60 * 60
        self.time = time.time()
        self.dirty = False
        self.cache = {}
        self.new = set()
        try:
            with open(self.fullpath) as file_:
                self.cache = json.load(file_)
        except (IOError, ValueError):
            pass

    def __del__(self):
        self.save()

{'fraction_non_alphanumeric': 0.10465793304221252, 'fraction_numerical': 0.008005822416302766, 'mean_word_length': 2.5619491964748575, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4851019', 'n_tokens_mistral': 2042, 'n_tokens_neox': 1947, 'n_words': 678}