<commit_before>import logging
import os
import requests
import yaml

from beets.plugins import BeetsPlugin
from beets import ui
from lxml import html

log = logging.getLogger('beets')

GENRES_TREE = os.path.join(os.path.dirname(__file__), 'genres-tree.yaml')

class RymGenrePlugin(BeetsPlugin):
    headers = {
         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
         'Accept-Language': 'en-US,en;q=0.8',
         'Cache-Control': 'max-age=0',
         'Connection': 'keep-alive',
         'Pragma': 'no-cache',
         'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31'
    }

    def __init__(self):
        super(RymGenrePlugin, self).__init__()

        self.config.add({
            'separator': u', ',
            'level': 'parent',
            'force': True,
            'auto': True
        })

        self.setup()

    def setup(self):
        def build_parents(elem, path, parents):
            if not path:
                path = []

            if isinstance(elem, dict):
                for (k, v) in elem.items():
                    build_parents(v, [k] + path, parents)
            elif isinstance(elem, list):
                for sub in elem:
                    build_parents(sub, path, parents)
            else:
                parents[elem] = path

        self.parent_genres = {}
        build_parents(yaml.load(open(GENRES_TREE, 'r')), [], self.parent_genres)

    def _get_albums(self, album):
        search_results = requests.post(
            'http://rateyourmusic.com/httprequest',
            data = {
                'searchterm': album['albumartist'] + " " + album['album'],
                'type': 'l',
                'page': 1,
                'action': 'Search',
                'rym_ajax_req': 1
            },
            headers = self.headers)

        def build_release(release_element):
            return {
                'artist': release_element.xpath('.//a[@class="artist"]/text()')[0],
                'album': release_element.xpath('.//a[@class="searchpage"]/text()')[0],
                'href': release_element.xpath('.//a[@class="searchpage"]/@href')[0]
            }

        return [build_release(release_element) for release_element in
                html.fromstring(search_results.text).xpath('//tr[@class="infobox"]')]

    def _get_genres(self, release_url):
        release_page = html.fromstring(requests.get(release_url, headers = self.headers).text)
        return release_page.xpath('//a[@class="genre"]/text()')

    def _get_genre(self, album):
        release = self._get_albums(album)[0]
        genres = self._get_genres('http://rateyourmusic.com/' + release['href'])
        genres_with_parents = set([])
        for genre in genres:
            genres_with_parents.add(genre)
            if genre in self.parent_genres:
                genres_with_parents |= set(self.parent_genres[genre])

        log.info(u'genres for album {0} - {1}: {2}'.format(
            album.albumartist,
            album.album,
            self.config['separator'].get(unicode).join(genres_with_parents)))

        return self.config['separator'].get(unicode).join(genres_with_parents)

    def commands(self):
        rymgenre_cmd = ui.Subcommand('rymgenre', help='fetch genres')
        def rymgenre_func(lib, opts, args):
            for album in lib.albums(ui.decargs(args)):
                album.genre = self._get_genre(album)
                album.store()

        rymgenre_cmd.func = rymgenre_func
        return [rymgenre_cmd]<commit_msg>Allow defining the level of genre-fetching<commit_after>import logging
import os
import requests
import yaml

from beets.plugins import BeetsPlugin
from beets import ui
from lxml import html

log = logging.getLogger('beets')

GENRES_TREE = os.path.join(os.path.dirname(__file__), 'genres-tree.yaml')

class RymGenrePlugin(BeetsPlugin):
    headers = {
         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
         'Accept-Language': 'en-US,en;q=0.8',
         'Cache-Control': 'max-age=0',
         'Connection': 'keep-alive',
         'Pragma': 'no-cache',
         'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31'
    }

    def __init__(self):
        super(RymGenrePlugin, self).__init__()

        self.config.add({
            'separator': u', ',
            'level': 'parent',
            'force': True,
            'auto': True
        })

        self.setup()

    def setup(self):
        def build_parents(elem, path, parents):
            if not path:
                path = []

            if isinstance(elem, dict):
                for (k, v) in elem.items():
                    build_parents(v, [k] + path, parents)
            elif isinstance(elem, list):
                for sub in elem:
                    build_parents(sub, path, parents)
            else:
                parents[elem] = path

        self.parent_genres = {}
        build_parents(yaml.load(open(GENRES_TREE, 'r')), [], self.parent_genres)

    def _get_albums(self, album):
        search_results = requests.post(
            'http://rateyourmusic.com/httprequest',
            data = {
                'searchterm': album['albumartist'] + " " + album['album'],
                'type': 'l',
                'page': 1,
                'action': 'Search',
                'rym_ajax_req': 1
            },
            headers = self.headers)

        def build_release(release_element):
            return {
                'artist': release_element.xpath('.//a[@class="artist"]/text()')[0],
                'album': release_element.xpath('.//a[@class="searchpage"]/text()')[0],
                'href': release_element.xpath('.//a[@class="searchpage"]/@href')[0]
            }

        return [build_release(release_element) for release_element in
                html.fromstring(search_results.text).xpath('//tr[@class="infobox"]')]

    def _get_genres(self, release_url):
        release_page = html.fromstring(requests.get(release_url, headers = self.headers).text)
        genres = { 'primary': [], 'secondary': [] }
        genres['primary'] = release_page.xpath('//span[@class="release_pri_genres"]//a[@class="genre"]/text()')
        genres['secondary'] = release_page.xpath('//span[@class="release_sec_genres"]//a[@class="genre"]/text()')
        return genres

    def _get_genre(self, album):
        release = self._get_albums(album)[0]
        genres = self._get_genres('http://rateyourmusic.com/' + release['href'])

        level = self.config['level'].as_choice(('primary', 'secondary', 'parent'))
        final_genres = set([])
        if level == 'primary':
            final_genres = set(genres['primary'])
        elif level == 'secondary':
            final_genres = set(genres['primary'] + genres['secondary'])
        elif level == 'parent':
            for genre in genres['primary'] + genres['secondary']:
                final_genres.add(genre)
                if genre in self.parent_genres:
                    final_genres |= set(self.parent_genres[genre])

        log.info(u'genres for album {0} - {1}: {2}'.format(
            album.albumartist,
            album.album,
            self.config['separator'].get(unicode).join(final_genres)))

        return self.config['separator'].get(unicode).join(final_genres)

    def commands(self):
        rymgenre_cmd = ui.Subcommand('rymgenre', help='fetch genres')
        def rymgenre_func(lib, opts, args):
            for album in lib.albums(ui.decargs(args)):
                album.genre = self._get_genre(album)
                album.store()

        rymgenre_cmd.func = rymgenre_func
        return [rymgenre_cmd]
{'fraction_non_alphanumeric': 0.14889867841409693, 'fraction_numerical': 0.012838263058527376, 'mean_word_length': 2.179671868747499, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '945663', 'n_tokens_mistral': 2566, 'n_tokens_neox': 2436, 'n_words': 438}