<commit_before>"""
Helper to download lyrics from lyrics.wikia.com
"""
from __future__ import unicode_literals
import re
import requests
from bs4 import BeautifulSoup, NavigableString, Tag
import logging


class Wikia:
    """Lyrics Downloader for lyrics.wikia.com"""
    url = "http://lyrics.wikia.com/api.php"

    def __init__(self):
        pass

    @staticmethod
    def parse(text, gracenote):
        """Parse lyrics from html"""
        # parse the result
        soup = BeautifulSoup(text)
        lyricbox = soup.find('div', "lyricbox")
        if lyricbox is None:
            logging.debug("BeautifulSoup doesn't find content")
            return None

        if gracenote:
            # gracenote lyrics are in a separate paragraph
            lyricbox = lyricbox.find('p')

        lyrics = ''
        for content in lyricbox.contents:
            if type(content) == NavigableString:
                lyrics += content.strip()
            elif type(content) == Tag:
                if content.string and content.name == "b" and content.string.startswith("Instrumental"):
                    return '{{Instrumental}}'
                elif content.name == "br":
                    lyrics += '\n'
                if content.string:
                    lyrics += content.string.strip()
        return lyrics.strip()

    @staticmethod
    def get_raw_data(artist, song):
        """Download html with lyrics, return None or tuple (text, gracenote)"""
        if not artist or not song:
            return None  # wikia needs both informations
        payload = {'artist': artist, 'song': song, 'fmt': "json"}
        result = requests.get(Wikia.url, params=payload)
        if result.status_code != 200:
            return None

        # The api returns a pseudo json object, that contains a url.
        match = re.search("'url':'([^']+)'", result.text)
        if match is None:
            return None

        html_url = match.group(1)
        logging.debug('fetch url %s', html_url)
        if 'action=edit' in html_url:
            logging.debug("no lyrics found")
            return None

        result = requests.get(html_url)
        gracenote = False

        if result.status_code != 200:
            # try it also with Gracenote: (e.g. <NAME> - High Hope)
            html_url = html_url[:9] + \
                html_url[9:].replace('/', '/Gracenote:', 1)
            logging.debug('fetch url %s', html_url)
            result = requests.get(html_url)
            gracenote = True
            if result.status_code != 200:
                return None
        return (result.text, gracenote)

    @staticmethod
    def fetch(artist, song, _):
        """Fetch lyrics from remote url"""
        data = Wikia.get_raw_data(artist, song)
        if data:
            return Wikia.parse(data[0], data[1])
        return None
<commit_msg>FIX BS4 bug with tags, another try<commit_after>"""
Helper to download lyrics from lyrics.wikia.com
"""
from __future__ import unicode_literals
import re
import requests
from bs4 import BeautifulSoup, NavigableString, Tag
import logging


class Wikia:
    """Lyrics Downloader for lyrics.wikia.com"""
    url = "http://lyrics.wikia.com/api.php"

    def __init__(self):
        pass

    @staticmethod
    def parse(text, gracenote):
        """Parse lyrics from html"""
        # parse the result
        soup = BeautifulSoup(text)
        lyricbox = soup.find('div', "lyricbox")
        if lyricbox is None:
            logging.debug("BeautifulSoup doesn't find content")
            return None

        if gracenote:
            # gracenote lyrics are in a separate paragraph
            lyricbox = lyricbox.find('p')

        lyrics = ''
        for content in lyricbox.contents:
            if type(content) == NavigableString:
                lyrics += content.strip()
            elif type(content) == Tag:
                if content.string and content.name == "b" and content.string.startswith("Instrumental"):
                    return '{{Instrumental}}'
                elif content.name == "br":
                    lyrics += '\n'
                if content.name in ['br', 'b', 'span'] and content.string:
                    lyrics += content.string.strip()
        return lyrics.strip()

    @staticmethod
    def get_raw_data(artist, song):
        """Download html with lyrics, return None or tuple (text, gracenote)"""
        if not artist or not song:
            return None  # wikia needs both informations
        payload = {'artist': artist, 'song': song, 'fmt': "json"}
        result = requests.get(Wikia.url, params=payload)
        if result.status_code != 200:
            return None

        # The api returns a pseudo json object, that contains a url.
        match = re.search("'url':'([^']+)'", result.text)
        if match is None:
            return None

        html_url = match.group(1)
        logging.debug('fetch url %s', html_url)
        if 'action=edit' in html_url:
            logging.debug("no lyrics found")
            return None

        result = requests.get(html_url)
        gracenote = False

        if result.status_code != 200:
            # try it also with Gracenote: (e.g. <NAME> - High Hope)
            html_url = html_url[:9] + \
                html_url[9:].replace('/', '/Gracenote:', 1)
            logging.debug('fetch url %s', html_url)
            result = requests.get(html_url)
            gracenote = True
            if result.status_code != 200:
                return None
        return (result.text, gracenote)

    @staticmethod
    def fetch(artist, song, _):
        """Fetch lyrics from remote url"""
        data = Wikia.get_raw_data(artist, song)
        if data:
            return Wikia.parse(data[0], data[1])
        return None

{'fraction_non_alphanumeric': 0.10870694131902371, 'fraction_numerical': 0.005712307425999654, 'mean_word_length': 1.944954128440367, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4414081', 'n_tokens_mistral': 1685, 'n_tokens_neox': 1612, 'n_words': 484}